preprocessing/pachter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scanpy==1.7.1 anndata==0.7.5 umap==0.4.6 numpy==1.19.4 scipy==1.5.4 pandas==1.1.4 scikit-learn==0.23.2 statsmodels==0.12.1 python-igraph==0.8.3 leidenalg==0.8.3\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import scanpy as sc\n",
    "import anndata as ad\n",
    "import pandas as pd\n",
    "import re\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "plt.rcParams['figure.figsize']=(5, 5)\n",
    "sc.settings.verbosity = 3\n",
    "sc.logging.print_header()\n",
    "os.chdir('./../')\n",
    "\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.set_figure_params(dpi=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## loading the raw data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/mohammad/Desktop/test_AdvAE'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mohammad/anaconda3/envs/scvi-env/lib/python3.7/site-packages/anndata/compat/__init__.py:182: FutureWarning: Moving element from .uns['neighbors']['distances'] to .obsp['distances'].\n",
      "\n",
      "This is where adjacency matrices should go now.\n",
      "  FutureWarning,\n",
      "/home/mohammad/anaconda3/envs/scvi-env/lib/python3.7/site-packages/anndata/compat/__init__.py:182: FutureWarning: Moving element from .uns['neighbors']['connectivities'] to .obsp['connectivities'].\n",
      "\n",
      "This is where adjacency matrices should go now.\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "adata = sc.read(\"./datasets/pachter_raw.h5ad\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_anna = sc.read(\"./datasets/pachter.h5ad\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "undoing some reindexing done by scanpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_index = []\n",
    "for obs in adata_anna.obs.index:\n",
    "    split = obs.split(\"-\")\n",
    "    new_index.append(split[0]+\"-\"+split[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_anna.obs.index = new_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(adata_anna.obs.index).intersection(adata.obs.index)) == len(adata_anna)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata = adata[new_index,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw = adata.raw.to_adata()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the original data is already normalized "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.53352"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw.X.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_new = sc.AnnData(X=raw.X, obs=adata_anna.obs, uns=adata_anna.uns, var=raw.var)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Xkr4', 'Gm1992', 'Rp1', 'Mrpl15', 'Lypla1', 'Gm37988', 'Tcea1',\n",
       "       'Rgs20', 'Atp6v1h', 'Rb1cc1',\n",
       "       ...\n",
       "       'Vamp7', 'Spry3', 'Tmlhe', 'AC132444.1', 'Csprs', 'AC125149.3',\n",
       "       'AC168977.1', 'PISD', 'DHRSX', 'CAAA01147332.1'],\n",
       "      dtype='object', name='index', length=16950)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata_new.var_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BMP</th>\n",
       "      <th>EGF</th>\n",
       "      <th>PlatePos</th>\n",
       "      <th>RA</th>\n",
       "      <th>ScripDec</th>\n",
       "      <th>batch</th>\n",
       "      <th>cell_type</th>\n",
       "      <th>cell_types</th>\n",
       "      <th>comb</th>\n",
       "      <th>condition</th>\n",
       "      <th>control</th>\n",
       "      <th>dose_val</th>\n",
       "      <th>louvain</th>\n",
       "      <th>n_counts</th>\n",
       "      <th>n_countslog</th>\n",
       "      <th>split</th>\n",
       "      <th>cov_drug_dose_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CTGATAGCAATGGACG-2</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>11</td>\n",
       "      <td>0R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0</td>\n",
       "      <td>11</td>\n",
       "      <td>2054.0</td>\n",
       "      <td>3.312600</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF_0.04+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CATCCACGTAGTGAAT-1</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>10</td>\n",
       "      <td>5R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0+1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>3832.0</td>\n",
       "      <td>3.583426</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.04+1.0+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AGTAGTCAGTGGAGTC-2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>14</td>\n",
       "      <td>0R,0.0B,0.8E,0S</td>\n",
       "      <td>EGF</td>\n",
       "      <td>1</td>\n",
       "      <td>0.04</td>\n",
       "      <td>14</td>\n",
       "      <td>6045.0</td>\n",
       "      <td>3.781396</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_EGF_0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGGGCGTAGAATTCCC-1</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>6</td>\n",
       "      <td>5R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0+1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>7974.0</td>\n",
       "      <td>3.901676</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.04+1.0+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGGCCAGCACATGACT-2</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>63</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>13</td>\n",
       "      <td>5R,4.0B,0.8E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.04+1.0</td>\n",
       "      <td>13</td>\n",
       "      <td>2576.0</td>\n",
       "      <td>3.410946</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+0.04+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGTATTCTCAGGTTCA-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+1.0+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>1730.0</td>\n",
       "      <td>3.238046</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+1.0+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTAGTTCCACAGATTC-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+1.0+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>1287.0</td>\n",
       "      <td>3.109579</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+1.0+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTCTCCTCACAGATTC-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>0R,4.0B,4.0E,1S</td>\n",
       "      <td>BMP+EGF+ScripDec</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>5099.0</td>\n",
       "      <td>3.707485</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+ScripDec_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGCGCCAAGACACG-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>3</td>\n",
       "      <td>0R,4.0B,4.0E,1S</td>\n",
       "      <td>BMP+EGF+ScripDec</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>3</td>\n",
       "      <td>6342.0</td>\n",
       "      <td>3.802226</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+ScripDec_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGGTTGTCCAGTAT-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,4.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>8049.0</td>\n",
       "      <td>3.905742</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19637 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    BMP   EGF  PlatePos  RA  ScripDec batch cell_type  \\\n",
       "CTGATAGCAATGGACG-2  0.8  20.0         8   0         0     0   unknown   \n",
       "CATCCACGTAGTGAAT-1  0.8  20.0        18   5         0     0   unknown   \n",
       "AGTAGTCAGTGGAGTC-2  0.0   0.8        59   0         0     0   unknown   \n",
       "TGGGCGTAGAATTCCC-1  0.8  20.0        18   5         0     0   unknown   \n",
       "TGGCCAGCACATGACT-2  4.0   0.8        63   5         0     0   unknown   \n",
       "...                 ...   ...       ...  ..       ...   ...       ...   \n",
       "TGTATTCTCAGGTTCA-4  4.0  20.0        16   1         0     2   unknown   \n",
       "TTAGTTCCACAGATTC-4  4.0  20.0        16   1         0     2   unknown   \n",
       "TTCTCCTCACAGATTC-4  4.0   4.0        28   0         1     2   unknown   \n",
       "TTTGCGCCAAGACACG-4  4.0   4.0        28   0         1     2   unknown   \n",
       "TTTGGTTGTCCAGTAT-4  4.0   4.0        40   1         0     2   unknown   \n",
       "\n",
       "                   cell_types              comb         condition  control  \\\n",
       "CTGATAGCAATGGACG-2         11  0R,0.8B,20.0E,0S           BMP+EGF        0   \n",
       "CATCCACGTAGTGAAT-1         10  5R,0.8B,20.0E,0S        BMP+EGF+RA        0   \n",
       "AGTAGTCAGTGGAGTC-2         14   0R,0.0B,0.8E,0S               EGF        1   \n",
       "TGGGCGTAGAATTCCC-1          6  5R,0.8B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TGGCCAGCACATGACT-2         13   5R,4.0B,0.8E,0S        BMP+EGF+RA        0   \n",
       "...                       ...               ...               ...      ...   \n",
       "TGTATTCTCAGGTTCA-4          5  1R,4.0B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TTAGTTCCACAGATTC-4          5  1R,4.0B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TTCTCCTCACAGATTC-4          5   0R,4.0B,4.0E,1S  BMP+EGF+ScripDec        0   \n",
       "TTTGCGCCAAGACACG-4          3   0R,4.0B,4.0E,1S  BMP+EGF+ScripDec        0   \n",
       "TTTGGTTGTCCAGTAT-4          5   1R,4.0B,4.0E,0S        BMP+EGF+RA        0   \n",
       "\n",
       "                        dose_val louvain  n_counts  n_countslog  split  \\\n",
       "CTGATAGCAATGGACG-2      0.04+1.0      11    2054.0     3.312600  train   \n",
       "CATCCACGTAGTGAAT-1  0.04+1.0+1.0      10    3832.0     3.583426  train   \n",
       "AGTAGTCAGTGGAGTC-2          0.04      14    6045.0     3.781396  train   \n",
       "TGGGCGTAGAATTCCC-1  0.04+1.0+1.0       6    7974.0     3.901676  train   \n",
       "TGGCCAGCACATGACT-2  0.2+0.04+1.0      13    2576.0     3.410946  train   \n",
       "...                          ...     ...       ...          ...    ...   \n",
       "TGTATTCTCAGGTTCA-4   0.2+1.0+0.2       5    1730.0     3.238046    ood   \n",
       "TTAGTTCCACAGATTC-4   0.2+1.0+0.2       5    1287.0     3.109579    ood   \n",
       "TTCTCCTCACAGATTC-4   0.2+0.2+0.2       5    5099.0     3.707485    ood   \n",
       "TTTGCGCCAAGACACG-4   0.2+0.2+0.2       3    6342.0     3.802226    ood   \n",
       "TTTGGTTGTCCAGTAT-4   0.2+0.2+0.2       5    8049.0     3.905742    ood   \n",
       "\n",
       "                                      cov_drug_dose_name  \n",
       "CTGATAGCAATGGACG-2              unknown_BMP+EGF_0.04+1.0  \n",
       "CATCCACGTAGTGAAT-1       unknown_BMP+EGF+RA_0.04+1.0+1.0  \n",
       "AGTAGTCAGTGGAGTC-2                      unknown_EGF_0.04  \n",
       "TGGGCGTAGAATTCCC-1       unknown_BMP+EGF+RA_0.04+1.0+1.0  \n",
       "TGGCCAGCACATGACT-2       unknown_BMP+EGF+RA_0.2+0.04+1.0  \n",
       "...                                                  ...  \n",
       "TGTATTCTCAGGTTCA-4        unknown_BMP+EGF+RA_0.2+1.0+0.2  \n",
       "TTAGTTCCACAGATTC-4        unknown_BMP+EGF+RA_0.2+1.0+0.2  \n",
       "TTCTCCTCACAGATTC-4  unknown_BMP+EGF+ScripDec_0.2+0.2+0.2  \n",
       "TTTGCGCCAAGACACG-4  unknown_BMP+EGF+ScripDec_0.2+0.2+0.2  \n",
       "TTTGGTTGTCCAGTAT-4        unknown_BMP+EGF+RA_0.2+0.2+0.2  \n",
       "\n",
       "[19637 rows x 17 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata_new.obs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# preprocessing "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keep the all genes in adata .raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_new.raw = adata_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalization and HVG selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "If you pass `n_top_genes`, all cutoffs are ignored.\n",
      "extracting highly variable genes\n",
      "    finished (0:00:00)\n",
      "--> added\n",
      "    'highly_variable', boolean vector (adata.var)\n",
      "    'means', float vector (adata.var)\n",
      "    'dispersions', float vector (adata.var)\n",
      "    'dispersions_norm', float vector (adata.var)\n"
     ]
    }
   ],
   "source": [
    "sc.pp.highly_variable_genes(adata_new,n_top_genes=5000, subset=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BMP</th>\n",
       "      <th>EGF</th>\n",
       "      <th>PlatePos</th>\n",
       "      <th>RA</th>\n",
       "      <th>ScripDec</th>\n",
       "      <th>batch</th>\n",
       "      <th>cell_type</th>\n",
       "      <th>cell_types</th>\n",
       "      <th>comb</th>\n",
       "      <th>condition</th>\n",
       "      <th>control</th>\n",
       "      <th>dose_val</th>\n",
       "      <th>louvain</th>\n",
       "      <th>n_counts</th>\n",
       "      <th>n_countslog</th>\n",
       "      <th>split</th>\n",
       "      <th>cov_drug_dose_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CTGATAGCAATGGACG-2</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>11</td>\n",
       "      <td>0R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0</td>\n",
       "      <td>11</td>\n",
       "      <td>2054.0</td>\n",
       "      <td>3.312600</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF_0.04+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CATCCACGTAGTGAAT-1</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>10</td>\n",
       "      <td>5R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0+1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>3832.0</td>\n",
       "      <td>3.583426</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.04+1.0+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AGTAGTCAGTGGAGTC-2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>14</td>\n",
       "      <td>0R,0.0B,0.8E,0S</td>\n",
       "      <td>EGF</td>\n",
       "      <td>1</td>\n",
       "      <td>0.04</td>\n",
       "      <td>14</td>\n",
       "      <td>6045.0</td>\n",
       "      <td>3.781396</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_EGF_0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGGGCGTAGAATTCCC-1</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>6</td>\n",
       "      <td>5R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0+1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>7974.0</td>\n",
       "      <td>3.901676</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.04+1.0+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGGCCAGCACATGACT-2</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>63</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>13</td>\n",
       "      <td>5R,4.0B,0.8E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.04+1.0</td>\n",
       "      <td>13</td>\n",
       "      <td>2576.0</td>\n",
       "      <td>3.410946</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+0.04+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGTATTCTCAGGTTCA-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+1.0+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>1730.0</td>\n",
       "      <td>3.238046</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+1.0+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTAGTTCCACAGATTC-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+1.0+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>1287.0</td>\n",
       "      <td>3.109579</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+1.0+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTCTCCTCACAGATTC-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>0R,4.0B,4.0E,1S</td>\n",
       "      <td>BMP+EGF+ScripDec</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>5099.0</td>\n",
       "      <td>3.707485</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+ScripDec_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGCGCCAAGACACG-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>3</td>\n",
       "      <td>0R,4.0B,4.0E,1S</td>\n",
       "      <td>BMP+EGF+ScripDec</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>3</td>\n",
       "      <td>6342.0</td>\n",
       "      <td>3.802226</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+ScripDec_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGGTTGTCCAGTAT-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,4.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>8049.0</td>\n",
       "      <td>3.905742</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19637 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    BMP   EGF  PlatePos  RA  ScripDec batch cell_type  \\\n",
       "CTGATAGCAATGGACG-2  0.8  20.0         8   0         0     0   unknown   \n",
       "CATCCACGTAGTGAAT-1  0.8  20.0        18   5         0     0   unknown   \n",
       "AGTAGTCAGTGGAGTC-2  0.0   0.8        59   0         0     0   unknown   \n",
       "TGGGCGTAGAATTCCC-1  0.8  20.0        18   5         0     0   unknown   \n",
       "TGGCCAGCACATGACT-2  4.0   0.8        63   5         0     0   unknown   \n",
       "...                 ...   ...       ...  ..       ...   ...       ...   \n",
       "TGTATTCTCAGGTTCA-4  4.0  20.0        16   1         0     2   unknown   \n",
       "TTAGTTCCACAGATTC-4  4.0  20.0        16   1         0     2   unknown   \n",
       "TTCTCCTCACAGATTC-4  4.0   4.0        28   0         1     2   unknown   \n",
       "TTTGCGCCAAGACACG-4  4.0   4.0        28   0         1     2   unknown   \n",
       "TTTGGTTGTCCAGTAT-4  4.0   4.0        40   1         0     2   unknown   \n",
       "\n",
       "                   cell_types              comb         condition  control  \\\n",
       "CTGATAGCAATGGACG-2         11  0R,0.8B,20.0E,0S           BMP+EGF        0   \n",
       "CATCCACGTAGTGAAT-1         10  5R,0.8B,20.0E,0S        BMP+EGF+RA        0   \n",
       "AGTAGTCAGTGGAGTC-2         14   0R,0.0B,0.8E,0S               EGF        1   \n",
       "TGGGCGTAGAATTCCC-1          6  5R,0.8B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TGGCCAGCACATGACT-2         13   5R,4.0B,0.8E,0S        BMP+EGF+RA        0   \n",
       "...                       ...               ...               ...      ...   \n",
       "TGTATTCTCAGGTTCA-4          5  1R,4.0B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TTAGTTCCACAGATTC-4          5  1R,4.0B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TTCTCCTCACAGATTC-4          5   0R,4.0B,4.0E,1S  BMP+EGF+ScripDec        0   \n",
       "TTTGCGCCAAGACACG-4          3   0R,4.0B,4.0E,1S  BMP+EGF+ScripDec        0   \n",
       "TTTGGTTGTCCAGTAT-4          5   1R,4.0B,4.0E,0S        BMP+EGF+RA        0   \n",
       "\n",
       "                        dose_val louvain  n_counts  n_countslog  split  \\\n",
       "CTGATAGCAATGGACG-2      0.04+1.0      11    2054.0     3.312600  train   \n",
       "CATCCACGTAGTGAAT-1  0.04+1.0+1.0      10    3832.0     3.583426  train   \n",
       "AGTAGTCAGTGGAGTC-2          0.04      14    6045.0     3.781396  train   \n",
       "TGGGCGTAGAATTCCC-1  0.04+1.0+1.0       6    7974.0     3.901676  train   \n",
       "TGGCCAGCACATGACT-2  0.2+0.04+1.0      13    2576.0     3.410946  train   \n",
       "...                          ...     ...       ...          ...    ...   \n",
       "TGTATTCTCAGGTTCA-4   0.2+1.0+0.2       5    1730.0     3.238046    ood   \n",
       "TTAGTTCCACAGATTC-4   0.2+1.0+0.2       5    1287.0     3.109579    ood   \n",
       "TTCTCCTCACAGATTC-4   0.2+0.2+0.2       5    5099.0     3.707485    ood   \n",
       "TTTGCGCCAAGACACG-4   0.2+0.2+0.2       3    6342.0     3.802226    ood   \n",
       "TTTGGTTGTCCAGTAT-4   0.2+0.2+0.2       5    8049.0     3.905742    ood   \n",
       "\n",
       "                                      cov_drug_dose_name  \n",
       "CTGATAGCAATGGACG-2              unknown_BMP+EGF_0.04+1.0  \n",
       "CATCCACGTAGTGAAT-1       unknown_BMP+EGF+RA_0.04+1.0+1.0  \n",
       "AGTAGTCAGTGGAGTC-2                      unknown_EGF_0.04  \n",
       "TGGGCGTAGAATTCCC-1       unknown_BMP+EGF+RA_0.04+1.0+1.0  \n",
       "TGGCCAGCACATGACT-2       unknown_BMP+EGF+RA_0.2+0.04+1.0  \n",
       "...                                                  ...  \n",
       "TGTATTCTCAGGTTCA-4        unknown_BMP+EGF+RA_0.2+1.0+0.2  \n",
       "TTAGTTCCACAGATTC-4        unknown_BMP+EGF+RA_0.2+1.0+0.2  \n",
       "TTCTCCTCACAGATTC-4  unknown_BMP+EGF+ScripDec_0.2+0.2+0.2  \n",
       "TTTGCGCCAAGACACG-4  unknown_BMP+EGF+ScripDec_0.2+0.2+0.2  \n",
       "TTTGGTTGTCCAGTAT-4        unknown_BMP+EGF+RA_0.2+0.2+0.2  \n",
       "\n",
       "[19637 rows x 17 columns]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata_new.obs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DE test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cpa.helper import rank_genes_groups_by_cov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Default of the method has been changed to 't-test' from 't-test_overestim_var'\n",
      "ranking genes\n",
      "Trying to set attribute `.uns` of view, copying.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unknown\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "    finished: added to `.uns['rank_genes_groups']`\n",
      "    'names', sorted np.recarray to be indexed by group ids\n",
      "    'scores', sorted np.recarray to be indexed by group ids\n",
      "    'logfoldchanges', sorted np.recarray to be indexed by group ids\n",
      "    'pvals', sorted np.recarray to be indexed by group ids\n",
      "    'pvals_adj', sorted np.recarray to be indexed by group ids (0:00:00)\n"
     ]
    }
   ],
   "source": [
    "rank_genes_groups_by_cov(adata_new, groupby='cov_drug_dose_name', covariate='cell_type', control_group='EGF_0.04', n_genes=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "saving the object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AnnData object with n_obs × n_vars = 19637 × 5000\n",
       "    obs: 'BMP', 'EGF', 'PlatePos', 'RA', 'ScripDec', 'batch', 'cell_type', 'cell_types', 'comb', 'condition', 'control', 'dose_val', 'louvain', 'n_counts', 'n_countslog', 'split', 'cov_drug_dose_name'\n",
       "    var: 'gene_ids', 'feature_types', 'genome', 'n_counts', 'highly_variable', 'means', 'dispersions', 'dispersions_norm'\n",
       "    uns: 'rank_genes_groups', 'rank_genes_groups_cov', 'hvg'"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BMP</th>\n",
       "      <th>EGF</th>\n",
       "      <th>PlatePos</th>\n",
       "      <th>RA</th>\n",
       "      <th>ScripDec</th>\n",
       "      <th>batch</th>\n",
       "      <th>cell_type</th>\n",
       "      <th>cell_types</th>\n",
       "      <th>comb</th>\n",
       "      <th>condition</th>\n",
       "      <th>control</th>\n",
       "      <th>dose_val</th>\n",
       "      <th>louvain</th>\n",
       "      <th>n_counts</th>\n",
       "      <th>n_countslog</th>\n",
       "      <th>split</th>\n",
       "      <th>cov_drug_dose_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CTGATAGCAATGGACG-2</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>11</td>\n",
       "      <td>0R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0</td>\n",
       "      <td>11</td>\n",
       "      <td>2054.0</td>\n",
       "      <td>3.312600</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF_0.04+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CATCCACGTAGTGAAT-1</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>10</td>\n",
       "      <td>5R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0+1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>3832.0</td>\n",
       "      <td>3.583426</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.04+1.0+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AGTAGTCAGTGGAGTC-2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>14</td>\n",
       "      <td>0R,0.0B,0.8E,0S</td>\n",
       "      <td>EGF</td>\n",
       "      <td>1</td>\n",
       "      <td>0.04</td>\n",
       "      <td>14</td>\n",
       "      <td>6045.0</td>\n",
       "      <td>3.781396</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_EGF_0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGGGCGTAGAATTCCC-1</th>\n",
       "      <td>0.8</td>\n",
       "      <td>20.0</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>6</td>\n",
       "      <td>5R,0.8B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04+1.0+1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>7974.0</td>\n",
       "      <td>3.901676</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.04+1.0+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGGCCAGCACATGACT-2</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>63</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>13</td>\n",
       "      <td>5R,4.0B,0.8E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.04+1.0</td>\n",
       "      <td>13</td>\n",
       "      <td>2576.0</td>\n",
       "      <td>3.410946</td>\n",
       "      <td>train</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+0.04+1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TGTATTCTCAGGTTCA-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+1.0+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>1730.0</td>\n",
       "      <td>3.238046</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+1.0+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTAGTTCCACAGATTC-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,20.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+1.0+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>1287.0</td>\n",
       "      <td>3.109579</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+1.0+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTCTCCTCACAGATTC-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>0R,4.0B,4.0E,1S</td>\n",
       "      <td>BMP+EGF+ScripDec</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>5099.0</td>\n",
       "      <td>3.707485</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+ScripDec_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGCGCCAAGACACG-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>3</td>\n",
       "      <td>0R,4.0B,4.0E,1S</td>\n",
       "      <td>BMP+EGF+ScripDec</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>3</td>\n",
       "      <td>6342.0</td>\n",
       "      <td>3.802226</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+ScripDec_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGGTTGTCCAGTAT-4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>1R,4.0B,4.0E,0S</td>\n",
       "      <td>BMP+EGF+RA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2+0.2+0.2</td>\n",
       "      <td>5</td>\n",
       "      <td>8049.0</td>\n",
       "      <td>3.905742</td>\n",
       "      <td>ood</td>\n",
       "      <td>unknown_BMP+EGF+RA_0.2+0.2+0.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19637 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    BMP   EGF  PlatePos  RA  ScripDec batch cell_type  \\\n",
       "CTGATAGCAATGGACG-2  0.8  20.0         8   0         0     0   unknown   \n",
       "CATCCACGTAGTGAAT-1  0.8  20.0        18   5         0     0   unknown   \n",
       "AGTAGTCAGTGGAGTC-2  0.0   0.8        59   0         0     0   unknown   \n",
       "TGGGCGTAGAATTCCC-1  0.8  20.0        18   5         0     0   unknown   \n",
       "TGGCCAGCACATGACT-2  4.0   0.8        63   5         0     0   unknown   \n",
       "...                 ...   ...       ...  ..       ...   ...       ...   \n",
       "TGTATTCTCAGGTTCA-4  4.0  20.0        16   1         0     2   unknown   \n",
       "TTAGTTCCACAGATTC-4  4.0  20.0        16   1         0     2   unknown   \n",
       "TTCTCCTCACAGATTC-4  4.0   4.0        28   0         1     2   unknown   \n",
       "TTTGCGCCAAGACACG-4  4.0   4.0        28   0         1     2   unknown   \n",
       "TTTGGTTGTCCAGTAT-4  4.0   4.0        40   1         0     2   unknown   \n",
       "\n",
       "                   cell_types              comb         condition  control  \\\n",
       "CTGATAGCAATGGACG-2         11  0R,0.8B,20.0E,0S           BMP+EGF        0   \n",
       "CATCCACGTAGTGAAT-1         10  5R,0.8B,20.0E,0S        BMP+EGF+RA        0   \n",
       "AGTAGTCAGTGGAGTC-2         14   0R,0.0B,0.8E,0S               EGF        1   \n",
       "TGGGCGTAGAATTCCC-1          6  5R,0.8B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TGGCCAGCACATGACT-2         13   5R,4.0B,0.8E,0S        BMP+EGF+RA        0   \n",
       "...                       ...               ...               ...      ...   \n",
       "TGTATTCTCAGGTTCA-4          5  1R,4.0B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TTAGTTCCACAGATTC-4          5  1R,4.0B,20.0E,0S        BMP+EGF+RA        0   \n",
       "TTCTCCTCACAGATTC-4          5   0R,4.0B,4.0E,1S  BMP+EGF+ScripDec        0   \n",
       "TTTGCGCCAAGACACG-4          3   0R,4.0B,4.0E,1S  BMP+EGF+ScripDec        0   \n",
       "TTTGGTTGTCCAGTAT-4          5   1R,4.0B,4.0E,0S        BMP+EGF+RA        0   \n",
       "\n",
       "                        dose_val louvain  n_counts  n_countslog  split  \\\n",
       "CTGATAGCAATGGACG-2      0.04+1.0      11    2054.0     3.312600  train   \n",
       "CATCCACGTAGTGAAT-1  0.04+1.0+1.0      10    3832.0     3.583426  train   \n",
       "AGTAGTCAGTGGAGTC-2          0.04      14    6045.0     3.781396  train   \n",
       "TGGGCGTAGAATTCCC-1  0.04+1.0+1.0       6    7974.0     3.901676  train   \n",
       "TGGCCAGCACATGACT-2  0.2+0.04+1.0      13    2576.0     3.410946  train   \n",
       "...                          ...     ...       ...          ...    ...   \n",
       "TGTATTCTCAGGTTCA-4   0.2+1.0+0.2       5    1730.0     3.238046    ood   \n",
       "TTAGTTCCACAGATTC-4   0.2+1.0+0.2       5    1287.0     3.109579    ood   \n",
       "TTCTCCTCACAGATTC-4   0.2+0.2+0.2       5    5099.0     3.707485    ood   \n",
       "TTTGCGCCAAGACACG-4   0.2+0.2+0.2       3    6342.0     3.802226    ood   \n",
       "TTTGGTTGTCCAGTAT-4   0.2+0.2+0.2       5    8049.0     3.905742    ood   \n",
       "\n",
       "                                      cov_drug_dose_name  \n",
       "CTGATAGCAATGGACG-2              unknown_BMP+EGF_0.04+1.0  \n",
       "CATCCACGTAGTGAAT-1       unknown_BMP+EGF+RA_0.04+1.0+1.0  \n",
       "AGTAGTCAGTGGAGTC-2                      unknown_EGF_0.04  \n",
       "TGGGCGTAGAATTCCC-1       unknown_BMP+EGF+RA_0.04+1.0+1.0  \n",
       "TGGCCAGCACATGACT-2       unknown_BMP+EGF+RA_0.2+0.04+1.0  \n",
       "...                                                  ...  \n",
       "TGTATTCTCAGGTTCA-4        unknown_BMP+EGF+RA_0.2+1.0+0.2  \n",
       "TTAGTTCCACAGATTC-4        unknown_BMP+EGF+RA_0.2+1.0+0.2  \n",
       "TTCTCCTCACAGATTC-4  unknown_BMP+EGF+ScripDec_0.2+0.2+0.2  \n",
       "TTTGCGCCAAGACACG-4  unknown_BMP+EGF+ScripDec_0.2+0.2+0.2  \n",
       "TTTGGTTGTCCAGTAT-4        unknown_BMP+EGF+RA_0.2+0.2+0.2  \n",
       "\n",
       "[19637 rows x 17 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata_new.obs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_new.write(\"./datasets/pachter_new.h5ad\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}