preprocessing/sciplex3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "satellite-immigration",
   "metadata": {},
   "outputs": [],
   "source": [
    "import scanpy as sc\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "distinguished-personal",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('./../')\n",
    "from cpa.helper import rank_genes_groups_by_cov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "previous-steal",
   "metadata": {},
   "outputs": [],
   "source": [
    "adatas = []\n",
    "for i in range(5):\n",
    "    adatas.append(sc.read(f'./datasets/sciplex_raw_chunk_{i}.h5ad'))\n",
    "adata = adatas[0].concatenate(adatas[1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "supposed-clinton",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "sc.pp.subsample(adata, fraction=0.5)\n",
    "sc.pp.normalize_per_cell(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "verified-reverse",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "sc.pp.log1p(adata)\n",
    "sc.pp.highly_variable_genes(adata, n_top_genes=5000, subset=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "sorted-proof",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-6-97db7d28ddbf>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs['dose_val'][adata.obs['product_name'].str.contains('Vehicle')] = 1.0\n",
      "<ipython-input-6-97db7d28ddbf>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs['product_name'][adata.obs['product_name'].str.contains('Vehicle')] = 'control'\n"
     ]
    }
   ],
   "source": [
    "adata.obs['dose_val'] = adata.obs.dose.astype(float) / np.max(adata.obs.dose.astype(float))\n",
    "adata.obs['dose_val'][adata.obs['product_name'].str.contains('Vehicle')] = 1.0\n",
    "adata.obs['product_name'] = [x.split(' ')[0] for x in adata.obs['product_name']]\n",
    "adata.obs['product_name'][adata.obs['product_name'].str.contains('Vehicle')] = 'control'\n",
    "adata.obs['drug_dose_name'] = adata.obs.product_name.astype(str) + '_' + adata.obs.dose_val.astype(str)\n",
    "adata.obs['cov_drug_dose_name'] = adata.obs.cell_type.astype(str) + '_' + adata.obs.drug_dose_name.astype(str)\n",
    "adata.obs['condition'] = adata.obs.product_name.copy()\n",
    "adata.obs['control'] = [1 if x == 'Vehicle_1.0' else 0 for x in adata.obs.drug_dose_name.values]\n",
    "adata.obs['cov_drug'] = adata.obs.cell_type.astype(str) + '_' + adata.obs.condition.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "renewable-insertion",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Default of the method has been changed to 't-test' from 't-test_overestim_var'\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/anndata/_core/anndata.py:1207: ImplicitModificationWarning: Initializing view as actual.\n",
      "  warnings.warn(\n",
      "Trying to set attribute `.obs` of view, copying.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "... storing 'cell_type' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'pathway' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'product_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'target' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'drug_dose_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'cov_drug_dose_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'condition' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'cov_drug' as categorical\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n",
      "WARNING: Default of the method has been changed to 't-test' from 't-test_overestim_var'\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/anndata/_core/anndata.py:1207: ImplicitModificationWarning: Initializing view as actual.\n",
      "  warnings.warn(\n",
      "Trying to set attribute `.obs` of view, copying.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MCF7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "... storing 'cell_type' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'pathway' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'product_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'target' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'drug_dose_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'cov_drug_dose_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'condition' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'cov_drug' as categorical\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n",
      "WARNING: Default of the method has been changed to 't-test' from 't-test_overestim_var'\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/anndata/_core/anndata.py:1207: ImplicitModificationWarning: Initializing view as actual.\n",
      "  warnings.warn(\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'cell_type' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "... storing 'pathway' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'product_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'target' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'drug_dose_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'cov_drug_dose_name' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'condition' as categorical\n",
      "Trying to set attribute `.obs` of view, copying.\n",
      "... storing 'cov_drug' as categorical\n"
     ]
    }
   ],
   "source": [
    "from cpa.helper import rank_genes_groups_by_cov\n",
    "rank_genes_groups_by_cov(adata, groupby='cov_drug', covariate='cell_type', control_group='control')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "latter-designer",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_genes_dict = {}\n",
    "for cat in adata.obs.cov_drug_dose_name.unique():\n",
    "    if 'control' not in cat:\n",
    "        rank_keys = np.array(list(adata.uns['rank_genes_groups_cov'].keys()))\n",
    "        bool_idx = [x in cat for x in rank_keys]\n",
    "        genes = adata.uns['rank_genes_groups_cov'][rank_keys[bool_idx][0]]\n",
    "        new_genes_dict[cat] = genes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "private-candy",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata.uns['rank_genes_groups_cov'] = new_genes_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advised-speed",
   "metadata": {},
   "source": [
    "# Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "australian-looking",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata.obs['split'] = 'train'  # reset\n",
    "ho_drugs = [\n",
    "    # selection of drugs from various pathways\n",
    "    \"Azacitidine\",\n",
    "    \"Carmofur\",\n",
    "    \"Pracinostat\",\n",
    "    \"Cediranib\",\n",
    "    \"Luminespib\",\n",
    "    \"Crizotinib\",\n",
    "    \"SNS-314\",\n",
    "    \"Obatoclax\",\n",
    "    \"Momelotinib\",\n",
    "    \"AG-14361\",\n",
    "    \"Entacapone\",\n",
    "    \"Fulvestrant\",\n",
    "    \"Mesna\",\n",
    "    \"Zileuton\",\n",
    "    \"Enzastaurin\",\n",
    "    \"IOX2\",\n",
    "    \"Alvespimycin\",\n",
    "    \"XAV-939\",\n",
    "    \"Fasudil\"\n",
    "]\n",
    "ood = adata.obs['condition'].isin(ho_drugs)\n",
    "len(ho_drugs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "gentle-cliff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-11-861cbc54850d>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs['split'][ood & (adata.obs['dose_val'] == 1.0)] = 'ood'\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    }
   ],
   "source": [
    "adata.obs['split'][ood & (adata.obs['dose_val'] == 1.0)] = 'ood'\n",
    "test_idx = sc.pp.subsample(adata[adata.obs['split'] != 'ood'], .10, copy=True).obs.index\n",
    "adata.obs['split'].loc[test_idx] = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ruled-mechanism",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>condition</th>\n",
       "      <th>(+)-JQ1</th>\n",
       "      <th>2-Methoxyestradiol</th>\n",
       "      <th>A-366</th>\n",
       "      <th>ABT-737</th>\n",
       "      <th>AC480</th>\n",
       "      <th>AG-14361</th>\n",
       "      <th>AG-490</th>\n",
       "      <th>AICAR</th>\n",
       "      <th>AMG-900</th>\n",
       "      <th>AR-42</th>\n",
       "      <th>...</th>\n",
       "      <th>Valproic</th>\n",
       "      <th>Vandetanib</th>\n",
       "      <th>Veliparib</th>\n",
       "      <th>WHI-P154</th>\n",
       "      <th>WP1066</th>\n",
       "      <th>XAV-939</th>\n",
       "      <th>YM155</th>\n",
       "      <th>ZM</th>\n",
       "      <th>Zileuton</th>\n",
       "      <th>control</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ood</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>355</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>249</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>403</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>168</td>\n",
       "      <td>141</td>\n",
       "      <td>170</td>\n",
       "      <td>133</td>\n",
       "      <td>165</td>\n",
       "      <td>137</td>\n",
       "      <td>189</td>\n",
       "      <td>173</td>\n",
       "      <td>149</td>\n",
       "      <td>156</td>\n",
       "      <td>...</td>\n",
       "      <td>192</td>\n",
       "      <td>142</td>\n",
       "      <td>152</td>\n",
       "      <td>180</td>\n",
       "      <td>190</td>\n",
       "      <td>120</td>\n",
       "      <td>40</td>\n",
       "      <td>134</td>\n",
       "      <td>127</td>\n",
       "      <td>664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>1338</td>\n",
       "      <td>1346</td>\n",
       "      <td>1511</td>\n",
       "      <td>1261</td>\n",
       "      <td>1490</td>\n",
       "      <td>1162</td>\n",
       "      <td>1559</td>\n",
       "      <td>1650</td>\n",
       "      <td>1187</td>\n",
       "      <td>1250</td>\n",
       "      <td>...</td>\n",
       "      <td>1612</td>\n",
       "      <td>1225</td>\n",
       "      <td>1445</td>\n",
       "      <td>1584</td>\n",
       "      <td>1661</td>\n",
       "      <td>1057</td>\n",
       "      <td>354</td>\n",
       "      <td>1260</td>\n",
       "      <td>1198</td>\n",
       "      <td>5800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 188 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "condition  (+)-JQ1  2-Methoxyestradiol  A-366  ABT-737  AC480  AG-14361  \\\n",
       "split                                                                     \n",
       "ood              0                   0      0        0      0       355   \n",
       "test           168                 141    170      133    165       137   \n",
       "train         1338                1346   1511     1261   1490      1162   \n",
       "\n",
       "condition  AG-490  AICAR  AMG-900  AR-42  ...  Valproic  Vandetanib  \\\n",
       "split                                     ...                         \n",
       "ood             0      0        0      0  ...         0           0   \n",
       "test          189    173      149    156  ...       192         142   \n",
       "train        1559   1650     1187   1250  ...      1612        1225   \n",
       "\n",
       "condition  Veliparib  WHI-P154  WP1066  XAV-939  YM155    ZM  Zileuton  \\\n",
       "split                                                                    \n",
       "ood                0         0       0      249      0     0       403   \n",
       "test             152       180     190      120     40   134       127   \n",
       "train           1445      1584    1661     1057    354  1260      1198   \n",
       "\n",
       "condition  control  \n",
       "split               \n",
       "ood              0  \n",
       "test           664  \n",
       "train         5800  \n",
       "\n",
       "[3 rows x 188 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(adata.obs['split'], adata.obs['condition'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "brown-marking",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train    256214\n",
       "test      28468\n",
       "ood        6206\n",
       "Name: split, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata.obs['split'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "outstanding-newport",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Fasudil         474\n",
       "Mesna           464\n",
       "IOX2            444\n",
       "Entacapone      433\n",
       "Fulvestrant     417\n",
       "Zileuton        403\n",
       "Azacitidine     385\n",
       "Carmofur        379\n",
       "Enzastaurin     366\n",
       "AG-14361        355\n",
       "Pracinostat     318\n",
       "SNS-314         280\n",
       "Crizotinib      256\n",
       "Momelotinib     249\n",
       "XAV-939         249\n",
       "Cediranib       248\n",
       "Obatoclax       195\n",
       "Luminespib      194\n",
       "Alvespimycin     97\n",
       "Name: condition, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata[adata.obs.split == 'ood'].obs.condition.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "after-family",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "control         664\n",
       "ENMD-2076       280\n",
       "MK-0752         202\n",
       "RG108           196\n",
       "Ramelteon       195\n",
       "               ... \n",
       "Luminespib       68\n",
       "Flavopiridol     68\n",
       "Patupilone       65\n",
       "Epothilone       56\n",
       "YM155            40\n",
       "Name: condition, Length: 188, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adata[adata.obs.split == 'test'].obs.condition.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "committed-cherry",
   "metadata": {},
   "source": [
    "Also a split which sees all data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "future-librarian",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    }
   ],
   "source": [
    "adata.obs['split_all'] = 'train'\n",
    "test_idx = sc.pp.subsample(adata, .10, copy=True).obs.index\n",
    "adata.obs['split_all'].loc[test_idx] = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "unable-cheese",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata.obs['ct_dose'] = adata.obs.cell_type.astype('str') + '_' + adata.obs.dose_val.astype('str')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "understood-battle",
   "metadata": {},
   "source": [
    "Round robin splits: dose and cell line combinations will be held out in turn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "prospective-mainland",
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "split_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "crazy-nylon",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-19-aaea970524bc>:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs[split_name][adata.obs.ct_dose == f'{ct}_{dose}'] = 'ood'\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train    229595\n",
       "test      43732\n",
       "ood       17561\n",
       "Name: split1, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    228515\n",
       "test      43526\n",
       "ood       18847\n",
       "Name: split2, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    229387\n",
       "test      43692\n",
       "ood       17809\n",
       "Name: split3, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    229731\n",
       "test      43758\n",
       "ood       17399\n",
       "Name: split4, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    214593\n",
       "test      40874\n",
       "ood       35421\n",
       "Name: split5, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    212368\n",
       "test      40450\n",
       "ood       38070\n",
       "Name: split6, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    213013\n",
       "test      40573\n",
       "ood       37302\n",
       "Name: split7, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    214820\n",
       "test      40917\n",
       "ood       35151\n",
       "Name: split8, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    229287\n",
       "test      43673\n",
       "ood       17928\n",
       "Name: split9, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    227678\n",
       "test      43367\n",
       "ood       19843\n",
       "Name: split10, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    228686\n",
       "test      43559\n",
       "ood       18643\n",
       "Name: split11, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    230139\n",
       "test      43835\n",
       "ood       16914\n",
       "Name: split12, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# single ct holdout\n",
    "for ct in adata.obs.cell_type.unique():\n",
    "    for dose in adata.obs.dose_val.unique():\n",
    "        i += 1\n",
    "        split_name = f'split{i}'\n",
    "        split_dict[split_name] = f'{ct}_{dose}'\n",
    "        \n",
    "        adata.obs[split_name] = 'train'\n",
    "        adata.obs[split_name][adata.obs.ct_dose == f'{ct}_{dose}'] = 'ood'\n",
    "        \n",
    "        test_idx = sc.pp.subsample(adata[adata.obs[split_name] != 'ood'], .16, copy=True).obs.index\n",
    "        adata.obs[split_name].loc[test_idx] = 'test'\n",
    "        \n",
    "        display(adata.obs[split_name].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "hundred-queue",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-20-3edecdb1cedf>:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs[split_name][adata.obs.ct_dose == f'{cts[0]}_{dose}'] = 'ood'\n",
      "<ipython-input-20-3edecdb1cedf>:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs[split_name][adata.obs.ct_dose == f'{cts[1]}_{dose}'] = 'ood'\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train    199842\n",
       "ood       52982\n",
       "test      38064\n",
       "Name: split13, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    196536\n",
       "ood       56917\n",
       "test      37435\n",
       "Name: split14, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    198053\n",
       "ood       55111\n",
       "test      37724\n",
       "Name: split15, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    200204\n",
       "ood       52550\n",
       "test      38134\n",
       "Name: split16, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    214536\n",
       "test      40863\n",
       "ood       35489\n",
       "Name: split17, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    211847\n",
       "test      40351\n",
       "ood       38690\n",
       "Name: split18, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    213727\n",
       "test      40709\n",
       "ood       36452\n",
       "Name: split19, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    215523\n",
       "test      41052\n",
       "ood       34313\n",
       "Name: split20, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    199533\n",
       "ood       53349\n",
       "test      38006\n",
       "Name: split21, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    195699\n",
       "ood       57913\n",
       "test      37276\n",
       "Name: split22, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    197353\n",
       "ood       55945\n",
       "test      37590\n",
       "Name: split23, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "train    200612\n",
       "ood       52065\n",
       "test      38211\n",
       "Name: split24, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# double ct holdout\n",
    "for cts in [('A549', 'MCF7'), ('A549', 'K562'), ('MCF7', 'K562')]:\n",
    "    for dose in adata.obs.dose_val.unique():\n",
    "        i += 1\n",
    "        split_name = f'split{i}'\n",
    "        split_dict[split_name] = f'{cts[0]}+{cts[1]}_{dose}'\n",
    "        \n",
    "        adata.obs[split_name] = 'train'\n",
    "        adata.obs[split_name][adata.obs.ct_dose == f'{cts[0]}_{dose}'] = 'ood'\n",
    "        adata.obs[split_name][adata.obs.ct_dose == f'{cts[1]}_{dose}'] = 'ood'\n",
    "        \n",
    "        test_idx = sc.pp.subsample(adata[adata.obs[split_name] != 'ood'], .16, copy=True).obs.index\n",
    "        adata.obs[split_name].loc[test_idx] = 'test'\n",
    "        \n",
    "        display(adata.obs[split_name].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "canadian-banking",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-21-58d929ed9378>:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs[split_name][adata.obs.dose_val == dose] = 'ood'\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train    184782\n",
       "ood       70910\n",
       "test      35196\n",
       "Name: split25, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-21-58d929ed9378>:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs[split_name][adata.obs.dose_val == dose] = 'ood'\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train    179868\n",
       "ood       76760\n",
       "test      34260\n",
       "Name: split26, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-21-58d929ed9378>:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs[split_name][adata.obs.dose_val == dose] = 'ood'\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train    182393\n",
       "ood       73754\n",
       "test      34741\n",
       "Name: split27, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-21-58d929ed9378>:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  adata.obs[split_name][adata.obs.dose_val == dose] = 'ood'\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/arrays/categorical.py:2487: FutureWarning: The `inplace` parameter in pandas.Categorical.remove_unused_categories is deprecated and will be removed in a future version.\n",
      "  res = method(*args, **kwargs)\n",
      "/home/icb/carlo.dedonno/anaconda3/envs/cpi2/lib/python3.8/site-packages/pandas/core/indexing.py:1637: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_block(indexer, value, name)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train    185997\n",
       "ood       69464\n",
       "test      35427\n",
       "Name: split28, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# triple ct holdout\n",
    "for dose in adata.obs.dose_val.unique():\n",
    "    i += 1\n",
    "    split_name = f'split{i}'\n",
    "\n",
    "    split_dict[split_name] = f'all_{dose}'\n",
    "    adata.obs[split_name] = 'train'\n",
    "    adata.obs[split_name][adata.obs.dose_val == dose] = 'ood'\n",
    "\n",
    "    test_idx = sc.pp.subsample(adata[adata.obs[split_name] != 'ood'], .16, copy=True).obs.index\n",
    "    adata.obs[split_name].loc[test_idx] = 'test'\n",
    "\n",
    "    display(adata.obs[split_name].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "executive-little",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata.uns['splits'] = split_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "arranged-trauma",
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.write('./datasets/sciplex3_new.h5ad', adata)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}