FeatureGeneration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pWxEMNAl1P3j"
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drugid</th>\n",
       "      <th>smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>CC(C)C[C@H](NC(=O)[C@@H](COC(C)(C)C)NC(=O)[C@H...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00035</td>\n",
       "      <td>NC(=O)CC[C@@H]1NC(=O)[C@H](CC2=CC=CC=C2)NC(=O)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00091</td>\n",
       "      <td>CCC1NC(=O)C(C(O)C(C)CC=CC)N(C)C(=O)C(C(C)C)N(C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00104</td>\n",
       "      <td>C[C@@H](O)[C@@H](CO)NC(=O)[C@@H]1CSSC[C@H](NC(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DB00114</td>\n",
       "      <td>CC1=NC=C(COP(O)(O)=O)C(C=O)=C1O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    drugid                                             smiles\n",
       "0  DB00014  CC(C)C[C@H](NC(=O)[C@@H](COC(C)(C)C)NC(=O)[C@H...\n",
       "1  DB00035  NC(=O)CC[C@@H]1NC(=O)[C@H](CC2=CC=CC=C2)NC(=O)...\n",
       "2  DB00091  CCC1NC(=O)C(C(O)C(C)CC=CC)N(C)C(=O)C(C(C)C)N(C...\n",
       "3  DB00104  C[C@@H](O)[C@@H](CO)NC(=O)[C@@H]1CSSC[C@H](NC(...\n",
       "4  DB00114                    CC1=NC=C(COP(O)(O)=O)C(C=O)=C1O"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_target = pd.read_csv(os.path.join(temp, drug_target))\n",
    "target_seq = pd.read_csv(os.path.join(temp, drug_seq))\n",
    "drug_goa = pd.read_csv(os.path.join(temp, drug_goa))\n",
    "drug_smiles = pd.read_csv(os.path.join(temp, drug_smiles))\n",
    "drug_se = pd.read_csv(os.path.join(temp, drug_se))\n",
    "\n",
    "#drug_target = pd.read_csv('data/input/drugbank-drug-target.csv')\n",
    "#target_seq = pd.read_csv('data/input/drugbank-target-seq.csv')\n",
    "#drug_goa = pd.read_csv('data/input/drugbank-drug-goa.csv')\n",
    "#drug_smiles = pd.read_csv('data/input/drugbank-drug-smiles.csv')\n",
    "#drug_se = pd.read_csv('data/input/drugbank-sider-se.csv')\n",
    "drug_smiles.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4hhOG_PYqWEW"
   },
   "source": [
    "## Get the drug data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 613,
     "status": "ok",
     "timestamp": 1520115966476,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "p0UNmiSis3Zz",
    "outputId": "8a98daf6-a068-4336-909d-ae100bf96a68"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579 drugs have all Target feature \n",
      "579 drugs have all Target GOA feature \n",
      "583 drugs have all Fingerprint feature \n",
      "524 drugs have all Sideeffect feature \n"
     ]
    }
   ],
   "source": [
    "print (\"%d drugs have all Target feature \"%len(  drug_target.drugid.unique()))\n",
    "print (\"%d drugs have Target GOA feature \"%len( drug_goa.drugid.unique()))\n",
    "print (\"%d drugs have Fingerprint feature \"%len(  drug_smiles.drugid.unique()))\n",
    "print (\"%d drugs have Sideeffect feature \"%len( drug_se.drugid.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 775,
     "status": "ok",
     "timestamp": 1520115970132,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "5AJASjxbtoyM",
    "outputId": "7028a6b4-af0f-4d60-98c7-53996546af3d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579 drugs have all Target SEQ feature \n"
     ]
    }
   ],
   "source": [
    "drug_target_seq = drug_target.merge(target_seq, on= ['geneid'])\n",
    "print (\"%d drugs have Target SEQ feature \"%len( drug_target_seq.drugid.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 616,
     "status": "ok",
     "timestamp": 1520115994596,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "t4oHFyCpu6Ez",
    "outputId": "2a2b220b-f9c4-481b-a41f-f88bce9c1956"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579 579 583 524\n",
      "510\n"
     ]
    }
   ],
   "source": [
    "a=drug_goa['drugid'].unique()\n",
    "b=drug_target['drugid'].unique()\n",
    "c=drug_smiles['drugid'].unique()\n",
    "d=drug_se['drugid'].unique()\n",
    "commonDrugs= set(a).intersection(b).intersection(c).intersection(d)\n",
    "print (len(a),len(b),len(c),len(d))\n",
    "print (len(commonDrugs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drugid</th>\n",
       "      <th>umlsid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00583</td>\n",
       "      <td>C0000729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00583</td>\n",
       "      <td>C0000737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00583</td>\n",
       "      <td>C0002418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00583</td>\n",
       "      <td>C0002871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DB00583</td>\n",
       "      <td>C0003123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    drugid    umlsid\n",
       "0  DB00583  C0000729\n",
       "1  DB00583  C0000737\n",
       "2  DB00583  C0002418\n",
       "3  DB00583  C0002871\n",
       "4  DB00583  C0003123"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_se.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZIFSdZDIx3nV"
   },
   "source": [
    "## Drug side effect similarity¶\n",
    "## calculating Jaccard coefficient based on drug sideefects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.mkdir(os.path.join(temp, 'features'))\n",
    "inter_folder= os.path.join(abs_path,'data/intermediate')\n",
    "if os.path.isdir(inter_folder):\n",
    "    shutil.rmtree(inter_folder)    \n",
    "os.mkdir(inter_folder)\n",
    "\n",
    "os.chdir(abs_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "B1b3149iv9k8"
   },
   "outputs": [],
   "source": [
    "drugSEDict = {k: g[\"umlsid\"].tolist() for k,g in drug_se.groupby(\"drugid\")}\n",
    "scores = []\n",
    "\n",
    "for comb in itertools.combinations(commonDrugs,2):\n",
    "    drug1 =comb[0]\n",
    "    drug2 =comb[1]\n",
    "\n",
    "    sideeffects1 = drugSEDict[drug1]\n",
    "    sideeffects2 = drugSEDict[drug2]\n",
    "    c = set(sideeffects1).intersection(sideeffects2)\n",
    "    u = set(sideeffects1).union(sideeffects2)\n",
    "    score = len(c)/float(len(u))\n",
    "    scores.append([drug1, drug2, score])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UvzACheczjX9"
   },
   "outputs": [],
   "source": [
    "drug_se_df = pd.DataFrame(scores, columns =['Drug1','Drug2','SE-SIM'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 535,
     "status": "ok",
     "timestamp": 1519610585922,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "DcGpavY_1dZ2",
    "outputId": "d6233046-5ed4-4582-a239-49e1776ee9b6"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Drug1</th>\n",
       "      <th>Drug2</th>\n",
       "      <th>SE-SIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB00366</td>\n",
       "      <td>0.085502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01029</td>\n",
       "      <td>0.131285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01217</td>\n",
       "      <td>0.145215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB00304</td>\n",
       "      <td>0.074890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01400</td>\n",
       "      <td>0.076000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Drug1    Drug2    SE-SIM\n",
       "0  DB00882  DB00366  0.085502\n",
       "1  DB00882  DB01029  0.131285\n",
       "2  DB00882  DB01217  0.145215\n",
       "3  DB00882  DB00304  0.074890\n",
       "4  DB00882  DB01400  0.076000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_se_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "drug_se_df.to_csv(os.path.join(temp,'features/drugs-se-sim.csv'), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tD672lWYgvhW"
   },
   "source": [
    "## PPI based drug-drug similarity\n",
    "###  calculate distance between drugs on protein-protein interaction network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install -q networkx==1.11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HL5hYg9Y1FRi"
   },
   "outputs": [],
   "source": [
    "# calcuate pairwise distance between proteins in the human PPI network\n",
    "import networkx as nx\n",
    "G= nx.Graph()\n",
    "with open(drug_ppi) as ppiFile: # human PPI network\n",
    "    next(ppiFile) # skip first line\n",
    "    drugs=set()\n",
    "    for line in ppiFile:\n",
    "        line=line.replace(\"'\",\"\").strip().split(',')\n",
    "        G.add_edge(line[0],line[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "ppi=nx.shortest_path_length(G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RgTWVD3ug1G0"
   },
   "outputs": [],
   "source": [
    "\n",
    "def grapDistance(ppi, target1, target2):\n",
    "    \"\"\"\n",
    "    return the shortest path between two proteins in the PPI network\n",
    "    ppi : dictonary that contains distance of PPI \n",
    "    target1 : first protein name\n",
    "    target2 : second protein name\n",
    "    \"\"\"\n",
    "    maxValue = 9999\n",
    "    if target1 not in ppi:\n",
    "        return maxValue\n",
    "    else:\n",
    "        if target2 not in ppi[target1]:\n",
    "            return maxValue\n",
    "        else:\n",
    "            return ppi[target1][target2]\n",
    "\n",
    "drug_targetlist = {k: g[\"geneid\"].tolist() for k,g in drug_target.groupby(\"drugid\")}\n",
    "values = []\n",
    "\n",
    "# calculate PPI-based pairwise drug similarity (Closeness)\n",
    "# First distances between proteins were transformed to similarity values using the formula described in Perlman et al (2011)\n",
    "# A, b were chosen according to Perlman et al (2011) to be 0.9 × e and 1, respectively.\n",
    "# Self similarity was assigned a value of 1.\n",
    "\n",
    "# For drugs similarities, maximal values between the two lists of associated genes were averaged \n",
    "# (taking into account both sides for symmetry).\n",
    "\n",
    "A = 0.9\n",
    "b = 1\n",
    "for comb in itertools.combinations(commonDrugs,2) :\n",
    "    drug1 = comb[0]\n",
    "    drug2 = comb[1]\n",
    "    if not(drug1 in drug_targetlist and drug2 in drug_targetlist) : continue\n",
    "    targetList1 = drug_targetlist[drug1]\n",
    "    targetList2 = drug_targetlist[drug2]\n",
    "    allscores =[]\n",
    "    for target1 in sorted(targetList1):\n",
    "        genescores = []\n",
    "        for target2 in sorted(targetList2):\n",
    "            target1 =str(target1)\n",
    "            target2 =str(target2)    \n",
    "            if target1 == target2:\n",
    "                score=1.0\n",
    "            else:\n",
    "                score = A*math.exp(-b* grapDistance(ppi, target1, target2))\n",
    "            genescores.append(score)\n",
    "    # add maximal values between the two lists of associated genes \n",
    "    allscores.append(max(genescores))\n",
    "    if len(allscores) ==0: continue\n",
    "    #average the maximal scores \n",
    "    maxScore =np.mean(allscores)\n",
    "    if maxScore >= 0:\n",
    "        values.append([drug1, drug2, maxScore])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NxvcGHHPjP2r"
   },
   "outputs": [],
   "source": [
    "drug_ppi_df = pd.DataFrame(values, columns =['Drug1','Drug2','PPI-SIM'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 672,
     "status": "ok",
     "timestamp": 1519678329835,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "Tmwrq2JruK1H",
    "outputId": "28653b27-75af-4e21-f8e8-ed85e41570ad"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Drug1</th>\n",
       "      <th>Drug2</th>\n",
       "      <th>PPI-SIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB00366</td>\n",
       "      <td>0.044808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01029</td>\n",
       "      <td>0.121802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01217</td>\n",
       "      <td>0.121802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB00304</td>\n",
       "      <td>0.331091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01400</td>\n",
       "      <td>0.044808</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Drug1    Drug2   PPI-SIM\n",
       "0  DB00882  DB00366  0.044808\n",
       "1  DB00882  DB01029  0.121802\n",
       "2  DB00882  DB01217  0.121802\n",
       "3  DB00882  DB00304  0.331091\n",
       "4  DB00882  DB01400  0.044808"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_ppi_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hYYrPmSW5q9b"
   },
   "outputs": [],
   "source": [
    "drug_ppi_df.to_csv(os.path.join(temp,'features/drugs-ppi-sim.csv'), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SLy8Cqv67Fjt"
   },
   "source": [
    "## Drug fingerprint similarity\n",
    "### calculating MACS based fingerprint (substructure) similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oddt import toolkit\n",
    "from oddt import fingerprints\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 725,
     "status": "ok",
     "timestamp": 1520035143730,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "aXAm6w0fGNCz",
    "outputId": "c266660c-5d82-46f5-a221-2f25c13d8655"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drugid</th>\n",
       "      <th>smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>CC(C)C[C@H](NC(=O)[C@@H](COC(C)(C)C)NC(=O)[C@H...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00035</td>\n",
       "      <td>NC(=O)CC[C@@H]1NC(=O)[C@H](CC2=CC=CC=C2)NC(=O)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00091</td>\n",
       "      <td>CCC1NC(=O)C(C(O)C(C)CC=CC)N(C)C(=O)C(C(C)C)N(C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00104</td>\n",
       "      <td>C[C@@H](O)[C@@H](CO)NC(=O)[C@@H]1CSSC[C@H](NC(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DB00122</td>\n",
       "      <td>C[N+](C)(C)CCO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    drugid                                             smiles\n",
       "0  DB00014  CC(C)C[C@H](NC(=O)[C@@H](COC(C)(C)C)NC(=O)[C@H...\n",
       "1  DB00035  NC(=O)CC[C@@H]1NC(=O)[C@H](CC2=CC=CC=C2)NC(=O)...\n",
       "2  DB00091  CCC1NC(=O)C(C(O)C(C)CC=CC)N(C)C(=O)C(C(C)C)N(C...\n",
       "3  DB00104  C[C@@H](O)[C@@H](CO)NC(=O)[C@@H]1CSSC[C@H](NC(...\n",
       "6  DB00122                                     C[N+](C)(C)CCO"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_smiles = drug_smiles[drug_smiles.drugid.isin(commonDrugs)]\n",
    "drug_smiles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drugid</th>\n",
       "      <th>smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>CC(C)C[C@H](NC(=O)[C@@H](COC(C)(C)C)NC(=O)[C@H...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00035</td>\n",
       "      <td>NC(=O)CC[C@@H]1NC(=O)[C@H](CC2=CC=CC=C2)NC(=O)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00091</td>\n",
       "      <td>CCC1NC(=O)C(C(O)C(C)CC=CC)N(C)C(=O)C(C(C)C)N(C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00104</td>\n",
       "      <td>C[C@@H](O)[C@@H](CO)NC(=O)[C@@H]1CSSC[C@H](NC(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DB00122</td>\n",
       "      <td>C[N+](C)(C)CCO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    drugid                                             smiles\n",
       "0  DB00014  CC(C)C[C@H](NC(=O)[C@@H](COC(C)(C)C)NC(=O)[C@H...\n",
       "1  DB00035  NC(=O)CC[C@@H]1NC(=O)[C@H](CC2=CC=CC=C2)NC(=O)...\n",
       "2  DB00091  CCC1NC(=O)C(C(O)C(C)CC=CC)N(C)C(=O)C(C(C)C)N(C...\n",
       "3  DB00104  C[C@@H](O)[C@@H](CO)NC(=O)[C@@H]1CSSC[C@H](NC(...\n",
       "6  DB00122                                     C[N+](C)(C)CCO"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_smiles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a dictionary of chemicals to be compared:\n",
    "input_dict = dict()\n",
    "for index,line in drug_smiles.iterrows():\n",
    "    id = line['drugid']\n",
    "    \n",
    "    smiles = line['smiles']\n",
    "    mol = toolkit.readstring(format='smiles',string=smiles)\n",
    "    fp =mol.calcfp(fptype='MACCS').raw\n",
    "    input_dict[id] = fp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tanimoto_score(fp1, fp2):\n",
    "    return np.sum(fp1 &  fp2) / np.sum(fp1 | fp2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_values=[]\n",
    "for chemical1, chemical2 in itertools.combinations(input_dict.keys(),2):\n",
    "    TC= tanimoto_score(input_dict[chemical1], input_dict[chemical2])\n",
    "    if chemical1 != chemical2:\n",
    "        sim_values.append([chemical1, chemical2, TC])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Drug1</th>\n",
       "      <th>Drug2</th>\n",
       "      <th>TC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>DB00035</td>\n",
       "      <td>0.732558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>DB00091</td>\n",
       "      <td>0.574713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>DB00104</td>\n",
       "      <td>0.655172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>DB00122</td>\n",
       "      <td>0.313953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>DB00125</td>\n",
       "      <td>0.361446</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Drug1    Drug2        TC\n",
       "0  DB00014  DB00035  0.732558\n",
       "1  DB00014  DB00091  0.574713\n",
       "2  DB00014  DB00104  0.655172\n",
       "3  DB00014  DB00122  0.313953\n",
       "4  DB00014  DB00125  0.361446"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chem_sim_df = pd.DataFrame(sim_values, columns=['Drug1','Drug2','TC'])\n",
    "chem_sim_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "chem_sim_df.to_csv(os.path.join(temp,'features/drugs-fingerprint-sim.csv'), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ngRHqprx2iBV"
   },
   "source": [
    "## Drug target sequence similarity\n",
    "### Calculation of SmithWaterman sequence alignment scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fasta2seq(lines):\n",
    "    lines = lines[lines.index('\\n')+1:]\n",
    "    lines =lines.replace('\\n','')\n",
    "    return lines\n",
    "\n",
    "target_seq.seq =target_seq.seq.map(fasta2seq)\n",
    "target_seq = target_seq[target_seq.geneid.isin(drug_target.geneid)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geneid</th>\n",
       "      <th>seq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1128</td>\n",
       "      <td>MNTSAPPAVSPNITVLAPGKGPWQVAFIGITTGLLSLATVTGNLLV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1129</td>\n",
       "      <td>MNNSTNSSNNSLALTSPYKTFEVVFIVLVAGSLSLVTIIGNILVMV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>148</td>\n",
       "      <td>MVFLSGNASDSSNCTQPPAPVNISKAILLGVILGGLILFGVLGNIL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>147</td>\n",
       "      <td>MNPDLDTGHNTSAPAHWGELKNANFTGPNQTSSNSTLPQLDITRAI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>146</td>\n",
       "      <td>MTFRDLLSVSFEGPRPDSSAGGSSAGGGGGSAGGAAPSEGPAVGGV...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   geneid                                                seq\n",
       "0    1128  MNTSAPPAVSPNITVLAPGKGPWQVAFIGITTGLLSLATVTGNLLV...\n",
       "1    1129  MNNSTNSSNNSLALTSPYKTFEVVFIVLVAGSLSLVTIIGNILVMV...\n",
       "2     148  MVFLSGNASDSSNCTQPPAPVNISKAILLGVILGGLILFGVLGNIL...\n",
       "3     147  MNPDLDTGHNTSAPAHWGELKNANFTGPNQTSSNSTLPQLDITRAI...\n",
       "4     146  MTFRDLLSVSFEGPRPDSSAGGSSAGGGGGSAGGAAPSEGPAVGGV..."
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_seq.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_seq_file=os.path.join(abs_path,\"data/intermediate/drugbank-target-seq-trimmed.tab\")\n",
    "target_seq.to_csv(target_seq_file,'\\t',index=False,header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SLF4J: Failed to load class \"org.slf4j.impl.StaticLoggerBinder\".\n",
      "SLF4J: Defaulting to no-operation (NOP) logger implementation\n",
      "SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.\n"
     ]
    }
   ],
   "source": [
    "target_seq_sim_file= os.path.join(abs_path, \"data/intermediate/target-target-seq-sim-biojava.tab\")\n",
    "\n",
    "!java -cp .:lib/smithwaterman.jar:lib/biojava-alignment-4.0.0.jar:lib/biojava-core-4.0.0.jar:lib/slf4j-api-1.7.10.jar biojava.targetseq.CalcLocalAlign {target_seq_file} > {target_seq_sim_file}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BG37vxz2qavr"
   },
   "outputs": [],
   "source": [
    "targetSeqSim=dict()\n",
    "with open(target_seq_sim_file) as tarSimfile:\n",
    "    for row in tarSimfile:\n",
    "        row = row.strip().split(\"\\t\")\n",
    "        t1 =row[0]\n",
    "        t2 = row[1]\n",
    "        sim = float(row[2])\n",
    "        targetSeqSim[(t1,t2)]=sim\n",
    "        targetSeqSim[(t2,t1)]=sim \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9_S9HbD92qrf"
   },
   "outputs": [],
   "source": [
    "\n",
    "drug_targetlist = {k: g[\"geneid\"].tolist() for k,g in drug_target_seq.groupby(\"drugid\")}\n",
    "values = []\n",
    "\n",
    "for comb in itertools.combinations(commonDrugs,2) :\n",
    "    drug1 = comb[0]\n",
    "    drug2 = comb[1]\n",
    "    if not(drug1 in drug_targetlist and drug2 in drug_targetlist) : continue\n",
    "    targetList1 = drug_targetlist[drug1]\n",
    "    targetList2 = drug_targetlist[drug2]\n",
    "    allscores =[]\n",
    "    for target1 in sorted(targetList1):\n",
    "        genescores = []\n",
    "        for target2 in sorted(targetList2):\n",
    "            target1 =str(target1)\n",
    "            target2 =str(target2)    \n",
    "            if target1 == target2:\n",
    "                score=1.0\n",
    "            else:\n",
    "                score = targetSeqSim[(target1,target2)] / (math.sqrt(targetSeqSim[(target1,target1)]) * math.sqrt(targetSeqSim[(target2,target2)]))\n",
    "            genescores.append(score)\n",
    "    # add maximal values between the two lists of associated genes \n",
    "    allscores.append(max(genescores))\n",
    "    if len(allscores) ==0: continue\n",
    "    #average the maximal scores \n",
    "    maxScore =np.mean(allscores)\n",
    "    values.append([drug1, drug2, maxScore])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bGwvGlaxrryF"
   },
   "outputs": [],
   "source": [
    "drug_seq_df = pd.DataFrame(values, columns =['Drug1','Drug2','TARGETSEQ-SIM'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 198
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 638,
     "status": "ok",
     "timestamp": 1519994837436,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -180
    },
    "id": "QJASSmD2vee3",
    "outputId": "0ac48671-efee-44a1-a30f-f95558ca10e9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Drug1</th>\n",
       "      <th>Drug2</th>\n",
       "      <th>TARGETSEQ-SIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB00366</td>\n",
       "      <td>0.033212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01029</td>\n",
       "      <td>0.045411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01217</td>\n",
       "      <td>0.043559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB00304</td>\n",
       "      <td>0.439197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01400</td>\n",
       "      <td>0.032739</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Drug1    Drug2  TARGETSEQ-SIM\n",
       "0  DB00882  DB00366       0.033212\n",
       "1  DB00882  DB01029       0.045411\n",
       "2  DB00882  DB01217       0.043559\n",
       "3  DB00882  DB00304       0.439197\n",
       "4  DB00882  DB01400       0.032739"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drug_seq_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "drug_seq_df.to_csv(os.path.join(temp,'features/drugs-target-seq-sim.csv'), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Gs4T317a4slx"
   },
   "source": [
    "## GO based drug-drug similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wzJYp-R05jfT"
   },
   "outputs": [],
   "source": [
    "drug_goa.drugid = drug_goa.drugid.map(lambda d: 'http://purl.obolibrary.org/obo/'+d)\n",
    "drug_goa.to_csv(os.path.join(abs_path,'data/intermediate/drug_goa.txt'),sep='\\t', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-ftK9iWl7V8z"
   },
   "outputs": [],
   "source": [
    "#cleaning GO annotations\n",
    "rows = []\n",
    "for comb in itertools.combinations(commonDrugs,2):\n",
    "    t1=comb[0]\n",
    "    t2=comb[1]\n",
    "    rows.append(['http://purl.obolibrary.org/obo/'+str(t1),'http://purl.obolibrary.org/obo/'+str(t2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mLV0_GQD8Qul"
   },
   "outputs": [],
   "source": [
    "drug_query_df = pd.DataFrame(rows, columns =['Drug1','Drug2'])\n",
    "drug_query_df.to_csv(os.path.join(abs_path,'data/intermediate/drug.gene.go.query'),sep='\\t', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "! if [ ! -f 'data/ontology/go.owl' ]; then gunzip 'data/ontology/go.owl'; fi "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PqA9J2QS9O02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------\n",
      "\tSemantic Measures Library Toolkit 0.9\n",
      "----------------------------------------------------------------------\n",
      "Toolkit dedicated to Semantic Measures computation and studies.\n",
      "Website: http://www.semantic-measures-library.org\n",
      "Developer: Sébastien Harispe\n",
      "\n",
      "Please cite: \n",
      "The Semantic Measures Library and Toolkit: \n",
      "fast computation of semantic similarity and relatedness \n",
      "using biomedical ontologies.\n",
      "Sébastien Harispe*, Sylvie Ranwez, Stefan Janaqi, Jacky Montmain\n",
      "Bioinformatics 2013; doi: 10.1093/bioinformatics/btt581\n",
      "----------------------------------------------------------------------\n",
      "Args [-t, sm]\n",
      "Args Tool    [-xmlconf, data/conf/sml.gene.go.conf]\n",
      "Loading SM Tool\n",
      "----------------------------------------------------------------------\n",
      "\tSemantic Measures Computer 0.9\n",
      "----------------------------------------------------------------------\n",
      "Tool used to compute Semantic similarity/relatedness considering an\n",
      "XML configuration file or a specific profile dedicated to an ontology.\n",
      "\n",
      "---------------------------------------------------------------\n",
      " Processing XML configuration data/conf/sml.gene.go.conf\n",
      "---------------------------------------------------------------\n",
      "Loading XML conf from : data/conf/sml.gene.go.conf\n",
      "Setting maximal number of threads to 4\n",
      "add namespace prefix : gene ref : http://purl.obolibrary.org/obo/gene_\n",
      "add namespace prefix : HP ref : http://purl.obolibrary.org/obo/HP_\n",
      "add namespace prefix : MP ref : http://purl.obolibrary.org/obo/MP_\n",
      "add namespace prefix : GO ref : http://purl.obolibrary.org/obo/GO_\n",
      "add namespace prefix : OMIM ref : http://bio2rdf.org/omim:\n",
      "add namespace prefix : DRUGBANK ref : http://bio2rdf.org/drugbank:\n",
      "generic configuration loaded \n",
      "Loading sm Configuration : data/conf/sml.gene.go.conf\n",
      "1 IC configurations loaded \n",
      "1 pairwise measure configurations loaded \n",
      "1 groupwise measure configurations  loaded \n",
      "Configuration loaded... \n",
      "Loading Graph http://purl.obolibrary.org/obo/gene_\n",
      "-------------------------------------\n",
      " Loading DATA\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      " RDF Loader\n",
      "-------------------------------------\n",
      "Populate graph http://purl.obolibrary.org/obo/gene_ from data/ontology/go.owl\n",
      "Parser loaded for: RDF/XML (mimeTypes=application/rdf+xml, application/xml; ext=rdf, rdfs, owl, xml)\n",
      "Parsing RDF file...\n",
      "Start Process\n",
      "100000 statements already loaded\n",
      "Number of vertices: 36175\n",
      "Number of edges   : 100000\n",
      "100000 statements already loaded\n",
      "Number of vertices: 36175\n",
      "Number of edges   : 100000\n",
      "Ending Process 126941 statements loaded \n",
      "vertices: 44117\n",
      "edges   : 126941\n",
      "Skipped (statement involving non URI ressources) : 1331598\n",
      "Graph http://purl.obolibrary.org/obo/gene_ populated by RDF data \n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "Loading Annotations using TSV loader.\n",
      "-------------------------------------\n",
      "file            data/intermediate/drug_goa.txt\n",
      "Skipping header false\n",
      "predicate       'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'\n",
      "\n",
      "\n",
      "[Error] Error processing file data/intermediate/drug_goa.txt\n",
      "data/intermediate/drug_goa.txt (No such file or directory)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### run the semantic relatedness library with given query and anotation file it will produce a file named: gene.go.sim.out\n",
    "!java -jar lib/sml-toolkit-0.9.jar -t sm -xmlconf data/conf/sml.gene.go.conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>e1</th>\n",
       "      <th>e2</th>\n",
       "      <th>bma</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://purl.obolibrary.org/obo/DB00882</td>\n",
       "      <td>http://purl.obolibrary.org/obo/DB00366</td>\n",
       "      <td>0.628361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://purl.obolibrary.org/obo/DB00882</td>\n",
       "      <td>http://purl.obolibrary.org/obo/DB01029</td>\n",
       "      <td>0.747916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://purl.obolibrary.org/obo/DB00882</td>\n",
       "      <td>http://purl.obolibrary.org/obo/DB01217</td>\n",
       "      <td>0.548093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://purl.obolibrary.org/obo/DB00882</td>\n",
       "      <td>http://purl.obolibrary.org/obo/DB00304</td>\n",
       "      <td>0.965051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://purl.obolibrary.org/obo/DB00882</td>\n",
       "      <td>http://purl.obolibrary.org/obo/DB01400</td>\n",
       "      <td>0.583699</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       e1  \\\n",
       "0  http://purl.obolibrary.org/obo/DB00882   \n",
       "1  http://purl.obolibrary.org/obo/DB00882   \n",
       "2  http://purl.obolibrary.org/obo/DB00882   \n",
       "3  http://purl.obolibrary.org/obo/DB00882   \n",
       "4  http://purl.obolibrary.org/obo/DB00882   \n",
       "\n",
       "                                       e2       bma  \n",
       "0  http://purl.obolibrary.org/obo/DB00366  0.628361  \n",
       "1  http://purl.obolibrary.org/obo/DB01029  0.747916  \n",
       "2  http://purl.obolibrary.org/obo/DB01217  0.548093  \n",
       "3  http://purl.obolibrary.org/obo/DB00304  0.965051  \n",
       "4  http://purl.obolibrary.org/obo/DB01400  0.583699  "
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "go_sim_df = pd.read_csv(os.path.join(abs_path,'data/intermediate/drug.gene.go.sim.out'),sep='\\t')\n",
    "go_sim_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "go_sim_df.rename(columns={'e1':'Drug1','e2':'Drug2','bma':'GO-SIM'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Drug1</th>\n",
       "      <th>Drug2</th>\n",
       "      <th>GO-SIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB00366</td>\n",
       "      <td>0.628361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01029</td>\n",
       "      <td>0.747916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01217</td>\n",
       "      <td>0.548093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB00304</td>\n",
       "      <td>0.965051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DB00882</td>\n",
       "      <td>DB01400</td>\n",
       "      <td>0.583699</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Drug1    Drug2    GO-SIM\n",
       "0  DB00882  DB00366  0.628361\n",
       "1  DB00882  DB01029  0.747916\n",
       "2  DB00882  DB01217  0.548093\n",
       "3  DB00882  DB00304  0.965051\n",
       "4  DB00882  DB01400  0.583699"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "go_sim_df.Drug1 = go_sim_df.Drug1.str.replace('http://purl.obolibrary.org/obo/','')\n",
    "go_sim_df.Drug2 = go_sim_df.Drug2.str.replace('http://purl.obolibrary.org/obo/','')\n",
    "go_sim_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "go_sim_df.to_csv(os.path.join(temp, 'features/drugs-target-go-sim.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WbTk5Qy-yrXo"
   },
   "source": [
    "## Disease Phenotype Similarity\n",
    "### MESH term based Similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oe9bgkmW-fs-"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drugid</th>\n",
       "      <th>omimid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB01148</td>\n",
       "      <td>231200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB01148</td>\n",
       "      <td>155100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DB01148</td>\n",
       "      <td>273800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DB00575</td>\n",
       "      <td>607554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DB00575</td>\n",
       "      <td>171300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    drugid  omimid\n",
       "0  DB01148  231200\n",
       "1  DB01148  155100\n",
       "2  DB01148  273800\n",
       "3  DB00575  607554\n",
       "4  DB00575  171300"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_df = pd.read_csv(drug_ind)\n",
    "predict_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_df.rename(columns={'drugid':'Drug','omimid':'Disease'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 695,
     "status": "ok",
     "timestamp": 1520028938756,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "NduMzqKV_Cx6",
    "outputId": "923e3e2e-0dcb-487c-ef68-53327d4ca8bd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gold std. diseases 313\n"
     ]
    }
   ],
   "source": [
    "gold_diseases = set( predict_df.Disease.unique())\n",
    "print ('Gold std. diseases',len(gold_diseases))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PeN6g9ET_TYx"
   },
   "outputs": [],
   "source": [
    "mesh_ann = {}\n",
    "allmeshterm = []\n",
    "with open(disease_mesh) as meshfile:\n",
    "    next(meshfile)\n",
    "    for line in meshfile:\n",
    "        line = line.strip().split(',')\n",
    "        if len(line) != 2: continue\n",
    "        di = line[0]\n",
    "        mesh = line[1].split(',')\n",
    "        mesh_ann[di]=mesh\n",
    "        allmeshterm.extend(mesh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AevY6bNZAyYH"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6166"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary = list(set(allmeshterm))\n",
    "len(vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0pvIICXSTyLm"
   },
   "outputs": [],
   "source": [
    "# create a co-occurrence matrix\n",
    "co_mat = np.zeros((len(mesh_ann),len(vocabulary)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3xl1Wpd5UAwE"
   },
   "outputs": [],
   "source": [
    "commonDiseases = mesh_ann.keys()\n",
    "mesh2id= { di:i for i,di in enumerate(mesh_ann.keys())}\n",
    "# fill in the co-occurrence matrix\n",
    "for key in mesh_ann:\n",
    "    annotations = mesh_ann[key]\n",
    "    col_index = [vocabulary.index(a) for a in annotations]\n",
    "    co_mat[mesh2id[key],col_index] =1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7t47ua0rXBHb"
   },
   "outputs": [],
   "source": [
    "def cosine_similarity(a,b):\n",
    "    return  np.dot(a, b)/(np.linalg.norm(a)*np.linalg.norm(b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8sQ5zv7UXy8A"
   },
   "outputs": [],
   "source": [
    "values = []\n",
    "# calculate cosine similarity between diseases using mesh annotation vector\n",
    "for comb in itertools.combinations(commonDiseases,2) :\n",
    "    disease1 = comb[0]\n",
    "    disease2 = comb[1]\n",
    "    sim = cosine_similarity(co_mat[mesh2id[disease1],:], co_mat[mesh2id[disease2],:])\n",
    "    values.append([disease1, disease2, sim])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bqENm65tYlMR"
   },
   "outputs": [],
   "source": [
    "disease_pheno_df = pd.DataFrame(values, columns =['Disease1','Disease2','PHENO-SIM'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 922,
     "status": "ok",
     "timestamp": 1520026556898,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "TfdZnbKkY74y",
    "outputId": "8c32ab95-2235-4240-e22a-01560c74affb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Disease1</th>\n",
       "      <th>Disease2</th>\n",
       "      <th>PHENO-SIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>231200</td>\n",
       "      <td>155100</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>231200</td>\n",
       "      <td>273800</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231200</td>\n",
       "      <td>607554</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>231200</td>\n",
       "      <td>171300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>231200</td>\n",
       "      <td>102300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Disease1 Disease2  PHENO-SIM\n",
       "0   231200   155100        0.0\n",
       "1   231200   273800        0.0\n",
       "2   231200   607554        0.0\n",
       "3   231200   171300        0.0\n",
       "4   231200   102300        0.0"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disease_pheno_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 724,
     "status": "ok",
     "timestamp": 1520026743982,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "CrjqBpglY-Hh",
    "outputId": "1ecad165-f4ae-453d-fa7a-ce4461bbef09"
   },
   "outputs": [],
   "source": [
    "disease_pheno_df.to_csv(os.path.join(temp, 'features/diseases-pheno-sim.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gt5PSUpsrk6C"
   },
   "source": [
    "## HPO based disease-disease similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1772,
     "status": "ok",
     "timestamp": 1520027951182,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "1628xjZJZBVS",
    "outputId": "4c42d61a-09b5-4f30-ac42-c0d5808b542f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diseaseid</th>\n",
       "      <th>hpoid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>231200</td>\n",
       "      <td>hpo:0000007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>231200</td>\n",
       "      <td>hpo:0002239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231200</td>\n",
       "      <td>hpo:0000132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>231200</td>\n",
       "      <td>hpo:0003010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>231200</td>\n",
       "      <td>hpo:0001873</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   diseaseid        hpoid\n",
       "0     231200  hpo:0000007\n",
       "1     231200  hpo:0002239\n",
       "2     231200  hpo:0000132\n",
       "3     231200  hpo:0003010\n",
       "4     231200  hpo:0001873"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading the hpo annotation file taken from compbio.charite.de/jennkins/jobs/hpo.annotations/\n",
    "disease_hpo = pd.read_csv(disease_hpo)\n",
    "disease_hpo.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "o2lhRNgA0TJP"
   },
   "outputs": [],
   "source": [
    "disease_hpo.rename(columns={'diseaseid':'Disease','hpoid':'HPO'}, inplace=True)\n",
    "disease_hpo.HPO= disease_hpo.HPO.str.replace('hpo','hp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 654,
     "status": "ok",
     "timestamp": 1520027956916,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "7bMDWYRrt2Cd",
    "outputId": "0effdcad-037a-4cc4-c17a-4c154a0e446e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Disease</th>\n",
       "      <th>HPO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>231200</td>\n",
       "      <td>hp:0000007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>231200</td>\n",
       "      <td>hp:0002239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231200</td>\n",
       "      <td>hp:0000132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>231200</td>\n",
       "      <td>hp:0003010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>231200</td>\n",
       "      <td>hp:0001873</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Disease         HPO\n",
       "0   231200  hp:0000007\n",
       "1   231200  hp:0002239\n",
       "2   231200  hp:0000132\n",
       "3   231200  hp:0003010\n",
       "4   231200  hp:0001873"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disease_hpo.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 654,
     "status": "ok",
     "timestamp": 1520028947780,
     "user": {
      "displayName": "Remzi Celebi",
      "photoUrl": "//lh5.googleusercontent.com/-Rp8CLE8Uv4I/AAAAAAAAAAI/AAAAAAAAQ0U/nP5lerOERdg/s50-c-k-no/photo.jpg",
      "userId": "107730920109755856468"
     },
     "user_tz": -60
    },
    "id": "cifgeNFFwyuF",
    "outputId": "2a828ac0-6648-412a-8d8b-c9576e5a4b09"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "310\n"
     ]
    }
   ],
   "source": [
    "diseasesWithFeatures= set(disease_hpo.Disease.unique()).intersection( gold_diseases )\n",
    "print (len(diseasesWithFeatures))\n",
    "rows = []\n",
    "for comb in itertools.combinations(diseasesWithFeatures,2):\n",
    "    t1=comb[0]\n",
    "    t2=comb[1]\n",
    "    rows.append(['omim:'+str(t1),'omim:'+str(t2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QYfGGj-90ehA"
   },
   "outputs": [],
   "source": [
    "disease_hpo[\"Disease\"]=disease_hpo[\"Disease\"].map(lambda d: 'omim:'+str(d))\n",
    "disease_hpo.to_csv(os.path.join(abs_path,'data/intermediate/disease_hpo.txt'), sep='\\t', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2QkjsRTfycHd"
   },
   "outputs": [],
   "source": [
    "disease_query_df = pd.DataFrame(rows, columns =['Disease1','Disease2'])\n",
    "disease_query_df.to_csv(os.path.join(abs_path, 'data/intermediate/hpo.sml.omim.query'), sep='\\t', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! if [ ! -f 'data/ontology/hpo.owl' ]; then gunzip 'data/ontology/hpo.owl'; fi "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2afDcubCuZwb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------\n",
      "\tSemantic Measures Library Toolkit 0.9\n",
      "----------------------------------------------------------------------\n",
      "Toolkit dedicated to Semantic Measures computation and studies.\n",
      "Website: http://www.semantic-measures-library.org\n",
      "Developer: Sébastien Harispe\n",
      "\n",
      "Please cite: \n",
      "The Semantic Measures Library and Toolkit: \n",
      "fast computation of semantic similarity and relatedness \n",
      "using biomedical ontologies.\n",
      "Sébastien Harispe*, Sylvie Ranwez, Stefan Janaqi, Jacky Montmain\n",
      "Bioinformatics 2013; doi: 10.1093/bioinformatics/btt581\n",
      "----------------------------------------------------------------------\n",
      "Args [-t, sm]\n",
      "Args Tool    [-xmlconf, data/conf/sml.omim.hpo.conf]\n",
      "Loading SM Tool\n",
      "----------------------------------------------------------------------\n",
      "\tSemantic Measures Computer 0.9\n",
      "----------------------------------------------------------------------\n",
      "Tool used to compute Semantic similarity/relatedness considering an\n",
      "XML configuration file or a specific profile dedicated to an ontology.\n",
      "\n",
      "---------------------------------------------------------------\n",
      " Processing XML configuration data/conf/sml.omim.hpo.conf\n",
      "---------------------------------------------------------------\n",
      "Loading XML conf from : data/conf/sml.omim.hpo.conf\n",
      "Setting maximal number of threads to 4\n",
      "add namespace prefix : mesh ref : http://bioonto.de/mesh.owl#\n",
      "add namespace prefix : gene ref : http://purl.obolibrary.org/obo/gene_\n",
      "add namespace prefix : HP ref : http://purl.obolibrary.org/obo/HP_\n",
      "add namespace prefix : MP ref : http://purl.obolibrary.org/obo/MP_\n",
      "add namespace prefix : GO ref : http://purl.obolibrary.org/obo/GO_\n",
      "add namespace prefix : OMIM ref : http://bio2rdf.org/omim:\n",
      "add namespace prefix : DRUGBANK ref : http://bio2rdf.org/drugbank:\n",
      "generic configuration loaded \n",
      "Loading sm Configuration : data/conf/sml.omim.hpo.conf\n",
      "1 IC configurations loaded \n",
      "1 pairwise measure configurations loaded \n",
      "1 groupwise measure configurations  loaded \n",
      "Configuration loaded... \n",
      "Loading Graph http://bio2rdf.org/omim:\n",
      "-------------------------------------\n",
      " Loading DATA\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      " RDF Loader\n",
      "-------------------------------------\n",
      "Populate graph http://bio2rdf.org/omim: from data/ontology/hpo.owl\n",
      "Parser loaded for: RDF/XML (mimeTypes=application/rdf+xml, application/xml; ext=rdf, rdfs, owl, xml)\n",
      "Parsing RDF file...\n",
      "Start Process\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "0 statements already loaded\n",
      "Number of vertices: 0\n",
      "Number of edges   : 0\n",
      "Ending Process 58190 statements loaded \n",
      "vertices: 16651\n",
      "edges   : 58190\n",
      "Skipped (statement involving non URI ressources) : 226165\n",
      "Graph http://bio2rdf.org/omim: populated by RDF data \n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "Loading Annotations using TSV loader.\n",
      "-------------------------------------\n",
      "file            data/intermediate/disease_hpo.txt\n",
      "Skipping header false\n",
      "predicate       'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'\n",
      "Number of lines skipped   0\n",
      "Number of lines processed 2832\n",
      "Number of statements loaded 2832\n",
      "TSV specification loaded.\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      " Applying actions\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "Rerooting\n",
      "-------------------------------------\n",
      "Fetching root node, uri: __FICTIVE__\n",
      "No root node explicitly specified using 'root_uri' parameter. Set root : http://www.w3.org/2002/07/owl#Thing\n",
      "Rooting taxonomic Graph using http://www.w3.org/2002/07/owl#Thing\n",
      "Number of roots 3624\n",
      "Starting process from 12128 vertices\n",
      "isDag : true\n",
      "Number of roots detected: 3624\n",
      "Rooting performed using http://www.w3.org/2002/07/owl#Thing as root 3623 edges created\n",
      "Number of roots 1\n",
      "isDag : true\n",
      "Rerooting performed\n",
      "-------------------------------------\n",
      "Actions performed\n",
      "-------------------------------------\n",
      "Retrieving the graph http://bio2rdf.org/omim:\n",
      "Graph information:\n",
      "http://bio2rdf.org/omim:\n",
      "Vertices\n",
      "\tTotal   : 16973  {e.g. http://purl.obolibrary.org/obo/HP_0011770}\n",
      "Edges \t  : 64645\n",
      "\n",
      "\n",
      "================================================================\n",
      "Loading Semantic Measures Engine for graph http://bio2rdf.org/omim:\n",
      "================================================================\n",
      "Graph Info: \n",
      "http://bio2rdf.org/omim:\n",
      "Vertices\n",
      "\tTotal   : 16973  {e.g. http://purl.obolibrary.org/obo/HP_0011770}\n",
      "Edges \t  : 64645\n",
      "\n",
      "\n",
      "---------------------------------------------------------------\n",
      "Pre-processing\n",
      "---------------------------------------------------------------\n",
      "Computing classes...\n",
      "Computing instances...\n",
      "Classes  : 16617\n",
      "Instances: 356\n",
      "---------------------------------------------------------------\n",
      "Inferences \n",
      "---------------------------------------------------------------\n",
      "Inferring ancestors...\n",
      "Checking Treatment coherency\n",
      "Incoherencies : 0\n",
      "Inferring descendants...\n",
      "Checking Treatment coherency\n",
      "Incoherencies : 0\n",
      "Inferring Conceptual Leaves...\n",
      "---------------------------------------------------------------\n",
      "Engine initialized\n",
      "================================================================\n",
      "---------------------------------------------------------------\n",
      "computing IC resnik\n",
      "---------------------------------------------------------------\n",
      "Class name slib.sml.sm.core.metrics.ic.annot.IC_annot_resnik_1995\n",
      "Checking null or infinite in the ICs computed\n",
      "ic resnik computed\n",
      "---------------------------------------------------------------\n",
      "Pairwise results caching set to false\n",
      "---------------------------------------------------------------\n",
      " Global parameters\n",
      "---------------------------------------------------------------\n",
      "quiet      : false\n",
      "Bench size : 5000\n",
      "Number of threads allowed: 4\n",
      "---------------------------------------------------------------\n",
      "checking DAG property\n",
      "Number of roots 1\n",
      "isDag : true\n",
      "---------------------------------------------------------------\n",
      "Start computation of 1 queries\n",
      "---------------------------------------------------------------\n",
      "Query :SM_computation\n",
      "---------------------------------------------------------------\n",
      "Query :SM_computation\n",
      "Not Found : SET\n",
      "score associated to entries for which an element is not found in the knowledge base: -1.0\n",
      "No Annotations : SET\n",
      "score associated to entities with no annotations : SET\n",
      "infile = data/intermediate/hpo.sml.omim.query\n",
      "outfile = data/intermediate/omim.hpo.sim.out\n",
      "type = oTOo\n",
      "use URI prefixes = false\n",
      "use URI prefixes (output)= false\n",
      "\n",
      "Loading Query file iterator: data/intermediate/hpo.sml.omim.query\n",
      "prefix (query file iterator): \n",
      "Starting computing query oTOo\n",
      "Number of query ~47896\n",
      "Create a pool worker of size 4\n",
      "processing queries...\n",
      "- 5000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 10000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 15000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 20000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 25000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 30000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 35000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 40000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 45000 / ~47896\tskipped 0\tsetted results 0\n",
      "- 47895 / ~47896\tskipped 0\tsetted results 0\n",
      "47895 queries considered\n",
      "skipped:0/47895(0%)\n",
      "setted :0/47895(0%)\n",
      "consult:data/intermediate/omim.hpo.sim.out\n",
      "process done\n",
      "---------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "### run the semantic relatedness library with given query and anotation file it will produce a file named: hpo.sim.out\n",
    "!java -jar lib/sml-toolkit-0.9.jar -t sm -xmlconf data/conf/sml.omim.hpo.conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "hpo_sim_df = pd.read_csv(os.path.join(abs_path,'data/intermediate/omim.hpo.sim.out'),sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>e1</th>\n",
       "      <th>e2</th>\n",
       "      <th>bma</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>omim:157950</td>\n",
       "      <td>omim:606798</td>\n",
       "      <td>0.373642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>omim:157950</td>\n",
       "      <td>omim:115300</td>\n",
       "      <td>0.344222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>omim:157950</td>\n",
       "      <td>omim:270960</td>\n",
       "      <td>0.355539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>omim:157950</td>\n",
       "      <td>omim:606842</td>\n",
       "      <td>0.322567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>omim:157950</td>\n",
       "      <td>omim:246400</td>\n",
       "      <td>0.343388</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            e1           e2       bma\n",
       "0  omim:157950  omim:606798  0.373642\n",
       "1  omim:157950  omim:115300  0.344222\n",
       "2  omim:157950  omim:270960  0.355539\n",
       "3  omim:157950  omim:606842  0.322567\n",
       "4  omim:157950  omim:246400  0.343388"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hpo_sim_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "hpo_sim_df.rename(columns={'e1':'Disease1','e2':'Disease2','bma':'HPO-SIM'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Disease1</th>\n",
       "      <th>Disease2</th>\n",
       "      <th>HPO-SIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>157950</td>\n",
       "      <td>606798</td>\n",
       "      <td>0.373642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>157950</td>\n",
       "      <td>115300</td>\n",
       "      <td>0.344222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>157950</td>\n",
       "      <td>270960</td>\n",
       "      <td>0.355539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>157950</td>\n",
       "      <td>606842</td>\n",
       "      <td>0.322567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>157950</td>\n",
       "      <td>246400</td>\n",
       "      <td>0.343388</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Disease1 Disease2   HPO-SIM\n",
       "0   157950   606798  0.373642\n",
       "1   157950   115300  0.344222\n",
       "2   157950   270960  0.355539\n",
       "3   157950   606842  0.322567\n",
       "4   157950   246400  0.343388"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hpo_sim_df.Disease1 = hpo_sim_df.Disease1.str.replace('omim:','')\n",
    "hpo_sim_df.Disease2 = hpo_sim_df.Disease2.str.replace('omim:','')\n",
    "hpo_sim_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "hpo_sim_df.to_csv(os.path.join(temp,'features/diseases-hpo-sim.csv'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Pipeline OpenPREDICT.ipynb",
   "provenance": [],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}