Benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1) Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Any, List, Union, Dict, Tuple\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import pathlib\n",
    "import time\n",
    "import subprocess\n",
    "import json\n",
    "import tqdm.auto as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIR = './data'\n",
    "MODELDIR = './models/'\n",
    "RESULTS_PATH = './results'\n",
    "BENCHMARKS = ['hg19/chr{}.fa'.format(i) for i in range(1,23) ] + ['hg38/chr1.fa', 'mm10/chr2.fa']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def benchmark(benchmarks: List[str], modelfiles: List[str], errorfile: str,\n",
    "              command: List[str],\n",
    "             results: Dict[str,Any]) -> Dict[str, Dict[str, float]]:\n",
    "    \"\"\"Benchmark a program\"\"\"\n",
    "    for k in tqdm.tqdm(benchmarks):\n",
    "        results.setdefault(k, {})\n",
    "        infile = pathlib.Path(DATADIR, k)\n",
    "        for model_path in tqdm.tqdm(modelfiles):\n",
    "            modelname = str(model_path.relative_to(MODELDIR))\n",
    "            if modelname in results[k]:\n",
    "                continue\n",
    "            outfile = '{}_{}.tsv'.format(k, modelname).replace('/', '_')\n",
    "            with open(outfile, 'wb') as file:\n",
    "                env = os.environ.copy()\n",
    "                env[\"TF_XLA_FLAGS\"]=\"--tf_xla_auto_jit=2\"\n",
    "                start_time = time.time()\n",
    "                process = subprocess.Popen(command + [model_path, infile],\n",
    "                                           stdout=file,\n",
    "                                           stderr=subprocess.PIPE,\n",
    "                                           env=env)\n",
    "                _, errdata = process.communicate()\n",
    "                end_time = time.time()\n",
    "            runtime = end_time - start_time\n",
    "            results[k][modelname] = {'runtime': runtime}\n",
    "            with open(errorfile, 'ab') as file:\n",
    "                file.write(errdata)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2) Run DeepGRP benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'deepgrp_runningtime.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert filename != 'deepgrp_runningtime.json', \"Change filename if you want to compute anything new\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with pathlib.Path(RESULTS_PATH, filename).open('r') as file:\n",
    "        results = json.load(file)\n",
    "except (json.JSONDecodeError, FileNotFoundError):\n",
    "    results = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "deepgrpmodels = list(pathlib.Path(MODELDIR).glob('model_*.h5'))\n",
    "deepgrp_command = ['python3', '-m', 'deepgrp', '-t 10', \"--xla\", \"-b 4096\"]\n",
    "deepgrp_results = benchmark(BENCHMARKS, deepgrpmodels, 'deepgrp.log',\n",
    "                            deepgrp_command, results = results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with pathlib.Path(RESULTS_PATH, filename).open('w') as file:\n",
    "    json.dump(deepgrp_results, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3) Run dna-brnn benchmark "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'dnabrnn_runningtime.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert filename != 'dnabrnn_runningtime.json', \"Change filename if you want to compute anything new\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with pathlib.Path(RESULTS_PATH, filename).open('r') as file:\n",
    "        results = json.load(file)\n",
    "except (json.JSONDecodeError, FileNotFoundError):\n",
    "    results = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dnabrnnmodels = list(pathlib.Path(MODELDIR).glob(\"dnabrnn_model*.knm\"))\n",
    "dnabrnn_command = ['dna-nn/dna-brnn', '-t 10', '-O292', '-Ai']\n",
    "dnabrnnresults = benchmark(BENCHMARKS, dnabrnnmodels, 'dnabrnn.log',\n",
    "                           dnabrnn_command, results = results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with pathlib.Path(RESULTS_PATH, filename).open('w') as file:\n",
    "    json.dump(dnabrnnresults, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:hydrogen"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}