dblp_parser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "from datetime import datetime\n",
    "import csv\n",
    "import codecs\n",
    "import ujson\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install ujson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#all of the element types in dblp\n",
    "all_elements = {\"article\", \"inproceedings\", \"proceedings\", \"book\", \"incollection\", \"phdthesis\", \n",
    "                \"mastersthesis\", \"www\"}\n",
    "# all of the feature types in dblp\n",
    "all_features = {\"address\", \"author\", \"booktitle\", \"cdrom\", \"chapter\", \"cite\", \"crossref\", \"editor\", \"ee\", \"isbn\",\n",
    "                \"journal\", \"month\", \"note\", \"number\", \"pages\", \"publisher\", \"school\", \"series\", \"title\", \"url\",\n",
    "                \"volume\", \"year\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_msg(message):\n",
    "    \"\"\"Produce a log with current time\"\"\"\n",
    "    print(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"), message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def context_iter(dblp_path):\n",
    "    \"\"\"Create a dblp data iterator of (event, element) pairs for processing\"\"\"\n",
    "    return etree.iterparse(source=dblp_path, dtd_validation=True, load_dtd=True)  # required dtd\n",
    "\n",
    "\n",
    "def clear_element(element):\n",
    "    \"\"\"Free up memory for temporary element tree after processing the element\"\"\"\n",
    "    element.clear()\n",
    "    while element.getprevious() is not None:\n",
    "        del element.getparent()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_feature(elem, features, include_key=False):\n",
    "    \"\"\"Extract the value of each feature\"\"\"\n",
    "    if include_key:\n",
    "        attribs = {'key': [elem.attrib['key']]}\n",
    "    else:\n",
    "        attribs = {}\n",
    "    for feature in features:\n",
    "        attribs[feature] = []\n",
    "    for sub in elem:\n",
    "        if sub.tag not in features:\n",
    "            continue\n",
    "        if sub.tag == 'title':\n",
    "            text = re.sub(\"<.*?>\", \"\", etree.tostring(sub).decode('utf-8')) if sub.text is None else sub.text\n",
    "        elif sub.tag == 'pages':\n",
    "            text = count_pages(sub.text)\n",
    "        else:\n",
    "            text = sub.text\n",
    "        if text is not None and len(text) > 0:\n",
    "            attribs[sub.tag] = attribs.get(sub.tag) + [text]\n",
    "    return attribs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_all(dblp_path, save_path, include_key=False):\n",
    "    log_msg(\"PROCESS: Start parsing...\")\n",
    "    f = open(save_path, 'w', encoding='utf8')\n",
    "    for _, elem in context_iter(dblp_path):\n",
    "        if elem.tag in all_elements:\n",
    "            attrib_values = extract_feature(elem, all_features, include_key)\n",
    "            f.write(str(attrib_values) + '\\n')\n",
    "        clear_element(elem)\n",
    "    f.close()\n",
    "    log_msg(\"FINISHED...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_author(dblp_path, save_path, save_to_csv=False):\n",
    "    type_name = ['article', 'book', 'incollection', 'inproceedings']\n",
    "    log_msg(\"PROCESS: Start parsing for {}...\".format(str(type_name)))\n",
    "    authors = set()\n",
    "    for _, elem in context_iter(dblp_path):\n",
    "        if elem.tag in type_name:\n",
    "            authors.update(a.text for a in elem.findall('author'))\n",
    "        elif elem.tag not in all_elements:\n",
    "            continue\n",
    "        clear_element(elem)\n",
    "    if save_to_csv:\n",
    "        f = open(save_path, 'w', newline='', encoding='utf8')\n",
    "        writer = csv.writer(f, delimiter=',')\n",
    "        writer.writerows([a] for a in sorted(authors))\n",
    "        f.close()\n",
    "    else:\n",
    "        with open(save_path, 'w', encoding='utf8') as f:\n",
    "            f.write('\\n'.join(sorted(authors)))\n",
    "    log_msg(\"FINISHED...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_inproceedings(dblp_path, save_path, save_to_csv=False, include_key=False):\n",
    "    type_name = [\"inproceedings\"]\n",
    "    features = ['title', 'author', 'year', 'pages', 'booktitle']\n",
    "    info = parse_entity(dblp_path, save_path, type_name, features, save_to_csv=save_to_csv, include_key=include_key)\n",
    "    log_msg('Total inproceedings found: {}, inproceedings contain all features: {}, inproceedings contain part of '\n",
    "            'features: {}'.format(info[0] + info[1], info[0], info[1]))\n",
    "    log_msg(\"Features information: {}\".format(str(info[2])))\n",
    "\n",
    "\n",
    "def parse_proceedings(dblp_path, save_path, save_to_csv=False, include_key=False):\n",
    "    type_name = [\"proceedings\"]\n",
    "    features = ['title', 'editor', 'year', 'booktitle', 'series', 'publisher']\n",
    "    # Other features are 'volume','isbn' and 'url'.\n",
    "    info = parse_entity(dblp_path, save_path, type_name, features, save_to_csv=save_to_csv, include_key=include_key)\n",
    "    log_msg('Total proceedings found: {}, proceedings contain all features: {}, proceedings contain part of '\n",
    "            'features: {}'.format(info[0] + info[1], info[0], info[1]))\n",
    "    log_msg(\"Features information: {}\".format(str(info[2])))\n",
    "\n",
    "\n",
    "def parse_book(dblp_path, save_path, save_to_csv=False, include_key=False):\n",
    "    type_name = [\"book\"]\n",
    "    features = ['title', 'author', 'publisher', 'isbn', 'year', 'pages']\n",
    "    info = parse_entity(dblp_path, save_path, type_name, features, save_to_csv=save_to_csv, include_key=include_key)\n",
    "    log_msg('Total books found: {}, books contain all features: {}, books contain part of features: {}'\n",
    "            .format(info[0] + info[1], info[0], info[1]))\n",
    "    log_msg(\"Features information: {}\".format(str(info[2])))\n",
    "\n",
    "\n",
    "def parse_publications(dblp_path, save_path, save_to_csv=False, include_key=False):\n",
    "    type_name = ['article', 'book', 'incollection', 'inproceedings']\n",
    "    features = ['title', 'author', 'year', 'journal', 'pages']\n",
    "    info = parse_entity(dblp_path, save_path, type_name, features, save_to_csv=save_to_csv, include_key=include_key)\n",
    "    log_msg('Total publications found: {}, publications contain all features: {}, publications contain part of '\n",
    "            'features: {}'.format(info[0] + info[1], info[0], info[1]))\n",
    "    log_msg(\"Features information: {}\".format(str(info[2])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_pages(pages):\n",
    "    cnt = 0\n",
    "    for part in re.compile(r\",\").split(pages):\n",
    "        subparts = re.compile(r\"-\").split(part)\n",
    "        if len(subparts) > 2:\n",
    "            continue\n",
    "        else:\n",
    "            try:\n",
    "                re_digits = re.compile(r\"[\\d]+\")\n",
    "                subparts = [int(re_digits.findall(sub)[-1]) for sub in subparts]\n",
    "            except IndexError:\n",
    "                continue\n",
    "            cnt += 1 if len(subparts) == 1 else subparts[1] - subparts[0] + 1\n",
    "    return \"\" if cnt == 0 else str(cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_entity(dblp_path, save_path, type_name, features=None, save_to_csv=False, include_key=False):\n",
    "    \"\"\"Parse specific elements according to the given type name and features\"\"\"\n",
    "    log_msg(\"PROCESS: Start parsing for {}...\".format(str(type_name)))\n",
    "    assert features is not None, \"features must be assigned before parsing the dblp dataset\"\n",
    "    results = []\n",
    "    attrib_count, full_entity, part_entity = {}, 0, 0\n",
    "    for _, elem in context_iter(dblp_path):\n",
    "        if elem.tag in type_name:\n",
    "            attrib_values = extract_feature(elem, features, include_key) # extract required features\n",
    "            results.append(attrib_values)  # add record to results array\n",
    "            for key, value in attrib_values.items():\n",
    "                attrib_count[key] = attrib_count.get(key, 0) + len(value)\n",
    "            cnt = sum([1 if len(x) > 0 else 0 for x in list(attrib_values.values())])\n",
    "            if cnt == len(features):\n",
    "                full_entity += 1\n",
    "            else:\n",
    "                part_entity += 1\n",
    "        elif elem.tag not in all_elements:\n",
    "            continue\n",
    "        clear_element(elem)\n",
    "    if save_to_csv:\n",
    "        f = open(save_path, 'w', newline='', encoding='utf8')\n",
    "        writer = csv.writer(f, delimiter=',')\n",
    "        writer.writerow(features)  # write title\n",
    "        for record in results:\n",
    "            # some features contain multiple values (e.g.: author), concatenate with `::`\n",
    "            row = ['::'.join(v) for v in list(record.values())]\n",
    "            writer.writerow(row)\n",
    "        f.close()\n",
    "    else:  # default save to json file\n",
    "        with codecs.open(save_path, mode='w', encoding='utf8', errors='ignore') as f:\n",
    "            ujson.dump(results, f)\n",
    "    return full_entity, part_entity, attrib_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_article(dblp_path, save_path, save_to_csv=False, include_key=False):\n",
    "    type_name = ['article']\n",
    "    features = ['title', 'author', 'year', 'journal', 'pages']\n",
    "    info = parse_entity(dblp_path, save_path, type_name, features, save_to_csv=save_to_csv, include_key=include_key)\n",
    "    log_msg('Total articles found: {}, articles contain all features: {}, articles contain part of features: {}'\n",
    "            .format(info[0] + info[1], info[0], info[1]))\n",
    "    log_msg(\"Features information: {}\".format(str(info[2])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    dblp_path = 'dblp/dblp.xml'\n",
    "    save_path = 'dblp/publications.csv'\n",
    "    try:\n",
    "        context_iter(dblp_path)\n",
    "        log_msg(\"LOG: Successfully loaded \\\"{}\\\".\".format(dblp_path))\n",
    "    except IOError:\n",
    "        log_msg(\"ERROR: Failed to load file \\\"{}\\\". Please check your XML and DTD files.\".format(dblp_path))\n",
    "        exit()\n",
    "    #parse_article(dblp_path, save_path, save_to_csv=False)\n",
    "    #parse_inproceedings(dblp_path, save_path, save_to_csv=False)\n",
    "    parse_publications(dblp_path, save_path, save_to_csv=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "dblp_path = 'dblp/dblp.xml'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sub_element:author, sub_text:Carmen Heine\n",
      "sub_element:title, sub_text:Modell zur Produktion von Online-Hilfen.\n",
      "sub_element:year, sub_text:2010\n",
      "sub_element:school, sub_text:Aarhus University\n",
      "sub_element:pages, sub_text:1-315\n",
      "sub_element:isbn, sub_text:978-3-86596-263-8\n",
      "sub_element:ee, sub_text:http://d-nb.info/996064095\n",
      "sub_element:author, sub_text:Gerd Hoff\n",
      "sub_element:title, sub_text:Ein Verfahren zur thematisch spezialisierten Suche im Web und seine Realisierung im Prototypen HomePageSearch\n",
      "sub_element:year, sub_text:2002\n",
      "sub_element:school, sub_text:University of Trier, Germany\n",
      "sub_element:ee, sub_text:http://ubt.opus.hbz-nrw.de/volltexte/2004/146/\n",
      "sub_element:ee, sub_text:http://nbn-resolving.de/urn:nbn:de:hbz:385-1468\n",
      "sub_element:ee, sub_text:http://d-nb.info/971713243\n"
     ]
    }
   ],
   "source": [
    "count=0\n",
    "for a, elem in context_iter(dblp_path):\n",
    "    count+=1\n",
    "    #print('Element Start')\n",
    "    #print(\"element key:{0}, element_tag:{1}, element_text:{2}\".format(elem.attrib.get('key'),elem.tag,elem.text))\n",
    "    #print('Element End')\n",
    "    for a in elem:\n",
    "        print(\"sub_element:{0}, sub_text:{1}\".format(a.tag,a.text))\n",
    "    if count==20:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "elem.attrib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}