tp1_b_mahalanobis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Voici les CIP des membres de l'équipe\n",
    "\n",
    "- Karl Plourde - PLOK2502"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "pd.set_option('display.max_rows', 20)\n",
    "pd.set_option('display.max_columns', 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>gene_0</th>\n",
       "      <th>gene_1</th>\n",
       "      <th>gene_2</th>\n",
       "      <th>gene_3</th>\n",
       "      <th>gene_4</th>\n",
       "      <th>gene_5</th>\n",
       "      <th>gene_6</th>\n",
       "      <th>gene_7</th>\n",
       "      <th>gene_8</th>\n",
       "      <th>...</th>\n",
       "      <th>gene_20522</th>\n",
       "      <th>gene_20523</th>\n",
       "      <th>gene_20524</th>\n",
       "      <th>gene_20525</th>\n",
       "      <th>gene_20526</th>\n",
       "      <th>gene_20527</th>\n",
       "      <th>gene_20528</th>\n",
       "      <th>gene_20529</th>\n",
       "      <th>gene_20530</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sample_0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.017209</td>\n",
       "      <td>3.265527</td>\n",
       "      <td>5.478487</td>\n",
       "      <td>10.431999</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.175175</td>\n",
       "      <td>0.591871</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.210257</td>\n",
       "      <td>9.723516</td>\n",
       "      <td>7.220030</td>\n",
       "      <td>9.119813</td>\n",
       "      <td>12.003135</td>\n",
       "      <td>9.650743</td>\n",
       "      <td>8.921326</td>\n",
       "      <td>5.286759</td>\n",
       "      <td>0.0</td>\n",
       "      <td>PRAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sample_1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.592732</td>\n",
       "      <td>1.588421</td>\n",
       "      <td>7.586157</td>\n",
       "      <td>9.623011</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.816049</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.323865</td>\n",
       "      <td>9.740931</td>\n",
       "      <td>6.256586</td>\n",
       "      <td>8.381612</td>\n",
       "      <td>12.674552</td>\n",
       "      <td>10.517059</td>\n",
       "      <td>9.397854</td>\n",
       "      <td>2.094168</td>\n",
       "      <td>0.0</td>\n",
       "      <td>LUAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sample_2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.511759</td>\n",
       "      <td>4.327199</td>\n",
       "      <td>6.881787</td>\n",
       "      <td>9.870730</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.972130</td>\n",
       "      <td>0.452595</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.127123</td>\n",
       "      <td>10.908640</td>\n",
       "      <td>5.401607</td>\n",
       "      <td>9.911597</td>\n",
       "      <td>9.045255</td>\n",
       "      <td>9.788359</td>\n",
       "      <td>10.090470</td>\n",
       "      <td>1.683023</td>\n",
       "      <td>0.0</td>\n",
       "      <td>PRAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sample_3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.663618</td>\n",
       "      <td>4.507649</td>\n",
       "      <td>6.659068</td>\n",
       "      <td>10.196184</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.843375</td>\n",
       "      <td>0.434882</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.792959</td>\n",
       "      <td>10.141520</td>\n",
       "      <td>8.942805</td>\n",
       "      <td>9.601208</td>\n",
       "      <td>11.392682</td>\n",
       "      <td>9.694814</td>\n",
       "      <td>9.684365</td>\n",
       "      <td>3.292001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>PRAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sample_4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.655741</td>\n",
       "      <td>2.821547</td>\n",
       "      <td>6.539454</td>\n",
       "      <td>9.738265</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.566967</td>\n",
       "      <td>0.360982</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.891425</td>\n",
       "      <td>10.373790</td>\n",
       "      <td>7.181162</td>\n",
       "      <td>9.846910</td>\n",
       "      <td>11.922439</td>\n",
       "      <td>9.217749</td>\n",
       "      <td>9.461191</td>\n",
       "      <td>5.110372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>BRCA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 20533 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID  gene_0    gene_1    gene_2    gene_3     gene_4  gene_5  \\\n",
       "0  sample_0     0.0  2.017209  3.265527  5.478487  10.431999     0.0   \n",
       "1  sample_1     0.0  0.592732  1.588421  7.586157   9.623011     0.0   \n",
       "2  sample_2     0.0  3.511759  4.327199  6.881787   9.870730     0.0   \n",
       "3  sample_3     0.0  3.663618  4.507649  6.659068  10.196184     0.0   \n",
       "4  sample_4     0.0  2.655741  2.821547  6.539454   9.738265     0.0   \n",
       "\n",
       "     gene_6    gene_7  gene_8  ...  gene_20522  gene_20523  gene_20524  \\\n",
       "0  7.175175  0.591871     0.0  ...    8.210257    9.723516    7.220030   \n",
       "1  6.816049  0.000000     0.0  ...    7.323865    9.740931    6.256586   \n",
       "2  6.972130  0.452595     0.0  ...    8.127123   10.908640    5.401607   \n",
       "3  7.843375  0.434882     0.0  ...    8.792959   10.141520    8.942805   \n",
       "4  6.566967  0.360982     0.0  ...    8.891425   10.373790    7.181162   \n",
       "\n",
       "   gene_20525  gene_20526  gene_20527  gene_20528  gene_20529  gene_20530  \\\n",
       "0    9.119813   12.003135    9.650743    8.921326    5.286759         0.0   \n",
       "1    8.381612   12.674552   10.517059    9.397854    2.094168         0.0   \n",
       "2    9.911597    9.045255    9.788359   10.090470    1.683023         0.0   \n",
       "3    9.601208   11.392682    9.694814    9.684365    3.292001         0.0   \n",
       "4    9.846910   11.922439    9.217749    9.461191    5.110372         0.0   \n",
       "\n",
       "   Class  \n",
       "0   PRAD  \n",
       "1   LUAD  \n",
       "2   PRAD  \n",
       "3   PRAD  \n",
       "4   BRCA  \n",
       "\n",
       "[5 rows x 20533 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data = pd.read_csv('data\\data.csv')\n",
    "df_labels = pd.read_csv('data\\labels.csv')\n",
    "\n",
    "#Merge data and labels\n",
    "df_merge = pd.merge(df_data, df_labels, left_index=True, right_index=True)\n",
    "df_merge.rename(columns={\"Unnamed: 0_x\":'ID'}, inplace=True)\n",
    "df_merge.drop(['Unnamed: 0_y'], axis=1, inplace=True)\n",
    "df_merge.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>gene_1</th>\n",
       "      <th>gene_2</th>\n",
       "      <th>gene_3</th>\n",
       "      <th>gene_4</th>\n",
       "      <th>gene_6</th>\n",
       "      <th>gene_11</th>\n",
       "      <th>gene_12</th>\n",
       "      <th>gene_19</th>\n",
       "      <th>gene_20</th>\n",
       "      <th>...</th>\n",
       "      <th>gene_20521</th>\n",
       "      <th>gene_20522</th>\n",
       "      <th>gene_20523</th>\n",
       "      <th>gene_20524</th>\n",
       "      <th>gene_20525</th>\n",
       "      <th>gene_20526</th>\n",
       "      <th>gene_20527</th>\n",
       "      <th>gene_20528</th>\n",
       "      <th>gene_20529</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sample_0</td>\n",
       "      <td>2.017209</td>\n",
       "      <td>3.265527</td>\n",
       "      <td>5.478487</td>\n",
       "      <td>10.431999</td>\n",
       "      <td>7.175175</td>\n",
       "      <td>1.334282</td>\n",
       "      <td>2.015391</td>\n",
       "      <td>5.619994</td>\n",
       "      <td>1.334282</td>\n",
       "      <td>...</td>\n",
       "      <td>4.926711</td>\n",
       "      <td>8.210257</td>\n",
       "      <td>9.723516</td>\n",
       "      <td>7.220030</td>\n",
       "      <td>9.119813</td>\n",
       "      <td>12.003135</td>\n",
       "      <td>9.650743</td>\n",
       "      <td>8.921326</td>\n",
       "      <td>5.286759</td>\n",
       "      <td>PRAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sample_1</td>\n",
       "      <td>0.592732</td>\n",
       "      <td>1.588421</td>\n",
       "      <td>7.586157</td>\n",
       "      <td>9.623011</td>\n",
       "      <td>6.816049</td>\n",
       "      <td>0.587845</td>\n",
       "      <td>2.466601</td>\n",
       "      <td>11.055208</td>\n",
       "      <td>3.562621</td>\n",
       "      <td>...</td>\n",
       "      <td>4.593372</td>\n",
       "      <td>7.323865</td>\n",
       "      <td>9.740931</td>\n",
       "      <td>6.256586</td>\n",
       "      <td>8.381612</td>\n",
       "      <td>12.674552</td>\n",
       "      <td>10.517059</td>\n",
       "      <td>9.397854</td>\n",
       "      <td>2.094168</td>\n",
       "      <td>LUAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sample_2</td>\n",
       "      <td>3.511759</td>\n",
       "      <td>4.327199</td>\n",
       "      <td>6.881787</td>\n",
       "      <td>9.870730</td>\n",
       "      <td>6.972130</td>\n",
       "      <td>0.452595</td>\n",
       "      <td>1.981122</td>\n",
       "      <td>8.210248</td>\n",
       "      <td>4.195285</td>\n",
       "      <td>...</td>\n",
       "      <td>5.125213</td>\n",
       "      <td>8.127123</td>\n",
       "      <td>10.908640</td>\n",
       "      <td>5.401607</td>\n",
       "      <td>9.911597</td>\n",
       "      <td>9.045255</td>\n",
       "      <td>9.788359</td>\n",
       "      <td>10.090470</td>\n",
       "      <td>1.683023</td>\n",
       "      <td>PRAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sample_3</td>\n",
       "      <td>3.663618</td>\n",
       "      <td>4.507649</td>\n",
       "      <td>6.659068</td>\n",
       "      <td>10.196184</td>\n",
       "      <td>7.843375</td>\n",
       "      <td>0.434882</td>\n",
       "      <td>2.874246</td>\n",
       "      <td>8.306317</td>\n",
       "      <td>3.573556</td>\n",
       "      <td>...</td>\n",
       "      <td>6.076566</td>\n",
       "      <td>8.792959</td>\n",
       "      <td>10.141520</td>\n",
       "      <td>8.942805</td>\n",
       "      <td>9.601208</td>\n",
       "      <td>11.392682</td>\n",
       "      <td>9.694814</td>\n",
       "      <td>9.684365</td>\n",
       "      <td>3.292001</td>\n",
       "      <td>PRAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sample_4</td>\n",
       "      <td>2.655741</td>\n",
       "      <td>2.821547</td>\n",
       "      <td>6.539454</td>\n",
       "      <td>9.738265</td>\n",
       "      <td>6.566967</td>\n",
       "      <td>1.275841</td>\n",
       "      <td>2.141204</td>\n",
       "      <td>10.149150</td>\n",
       "      <td>2.967630</td>\n",
       "      <td>...</td>\n",
       "      <td>5.996032</td>\n",
       "      <td>8.891425</td>\n",
       "      <td>10.373790</td>\n",
       "      <td>7.181162</td>\n",
       "      <td>9.846910</td>\n",
       "      <td>11.922439</td>\n",
       "      <td>9.217749</td>\n",
       "      <td>9.461191</td>\n",
       "      <td>5.110372</td>\n",
       "      <td>BRCA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 16963 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID    gene_1    gene_2    gene_3     gene_4    gene_6   gene_11  \\\n",
       "0  sample_0  2.017209  3.265527  5.478487  10.431999  7.175175  1.334282   \n",
       "1  sample_1  0.592732  1.588421  7.586157   9.623011  6.816049  0.587845   \n",
       "2  sample_2  3.511759  4.327199  6.881787   9.870730  6.972130  0.452595   \n",
       "3  sample_3  3.663618  4.507649  6.659068  10.196184  7.843375  0.434882   \n",
       "4  sample_4  2.655741  2.821547  6.539454   9.738265  6.566967  1.275841   \n",
       "\n",
       "    gene_12    gene_19   gene_20  ...  gene_20521  gene_20522  gene_20523  \\\n",
       "0  2.015391   5.619994  1.334282  ...    4.926711    8.210257    9.723516   \n",
       "1  2.466601  11.055208  3.562621  ...    4.593372    7.323865    9.740931   \n",
       "2  1.981122   8.210248  4.195285  ...    5.125213    8.127123   10.908640   \n",
       "3  2.874246   8.306317  3.573556  ...    6.076566    8.792959   10.141520   \n",
       "4  2.141204  10.149150  2.967630  ...    5.996032    8.891425   10.373790   \n",
       "\n",
       "   gene_20524  gene_20525  gene_20526  gene_20527  gene_20528  gene_20529  \\\n",
       "0    7.220030    9.119813   12.003135    9.650743    8.921326    5.286759   \n",
       "1    6.256586    8.381612   12.674552   10.517059    9.397854    2.094168   \n",
       "2    5.401607    9.911597    9.045255    9.788359   10.090470    1.683023   \n",
       "3    8.942805    9.601208   11.392682    9.694814    9.684365    3.292001   \n",
       "4    7.181162    9.846910   11.922439    9.217749    9.461191    5.110372   \n",
       "\n",
       "   Class  \n",
       "0   PRAD  \n",
       "1   LUAD  \n",
       "2   PRAD  \n",
       "3   PRAD  \n",
       "4   BRCA  \n",
       "\n",
       "[5 rows x 16963 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Calculate the threshold for the percentage of zeros\n",
    "threshold = 0.3 \n",
    "\n",
    "# Calculate the maximum allowed number of zeros\n",
    "max_zeros = len(df_merge) * threshold\n",
    "\n",
    "# Filter columns with 80% or more zeros\n",
    "filtered_columns = df_merge.columns[(df_merge == 0).sum() <= max_zeros]\n",
    "\n",
    "# Create a new DataFrame with selected columns\n",
    "filtered_data = df_merge[filtered_columns]\n",
    "\n",
    "# Display the modified DataFrame\n",
    "filtered_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Make a dataframe for each value of PRAD LUAD BRCA KIRC COAD\n",
    "df_prad = filtered_data[filtered_data['Class'] == 'PRAD']\n",
    "df_luad = filtered_data[filtered_data['Class'] == 'LUAD']\n",
    "df_brca= filtered_data[filtered_data['Class'] == 'BRCA']\n",
    "df_kirc = filtered_data[filtered_data['Class'] == 'KIRC']\n",
    "df_coad = filtered_data[filtered_data['Class'] == 'COAD']\n",
    "\n",
    "#Remove Class column and ID column\n",
    "df_coad= df_coad.iloc[:,1:-1]\n",
    "\n",
    "df_kirc= df_kirc.iloc[:,1:-1]\n",
    "\n",
    "df_brca= df_brca.iloc[:,1:-1]\n",
    "\n",
    "df_luad= df_luad.iloc[:,1:-1]\n",
    "\n",
    "df_prad = df_prad.iloc[:,1:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_1</th>\n",
       "      <th>gene_2</th>\n",
       "      <th>gene_3</th>\n",
       "      <th>gene_4</th>\n",
       "      <th>gene_6</th>\n",
       "      <th>gene_11</th>\n",
       "      <th>gene_12</th>\n",
       "      <th>gene_19</th>\n",
       "      <th>gene_20</th>\n",
       "      <th>gene_22</th>\n",
       "      <th>...</th>\n",
       "      <th>gene_20515</th>\n",
       "      <th>gene_20516</th>\n",
       "      <th>gene_20518</th>\n",
       "      <th>gene_20519</th>\n",
       "      <th>gene_20520</th>\n",
       "      <th>gene_20521</th>\n",
       "      <th>gene_20524</th>\n",
       "      <th>gene_20527</th>\n",
       "      <th>gene_20528</th>\n",
       "      <th>gene_20529</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.017209</td>\n",
       "      <td>3.265527</td>\n",
       "      <td>5.478487</td>\n",
       "      <td>10.431999</td>\n",
       "      <td>7.175175</td>\n",
       "      <td>1.334282</td>\n",
       "      <td>2.015391</td>\n",
       "      <td>5.619994</td>\n",
       "      <td>1.334282</td>\n",
       "      <td>9.796088</td>\n",
       "      <td>...</td>\n",
       "      <td>7.175175</td>\n",
       "      <td>9.249369</td>\n",
       "      <td>8.045563</td>\n",
       "      <td>7.475709</td>\n",
       "      <td>7.205236</td>\n",
       "      <td>4.926711</td>\n",
       "      <td>7.220030</td>\n",
       "      <td>9.650743</td>\n",
       "      <td>8.921326</td>\n",
       "      <td>5.286759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.511759</td>\n",
       "      <td>4.327199</td>\n",
       "      <td>6.881787</td>\n",
       "      <td>9.870730</td>\n",
       "      <td>6.972130</td>\n",
       "      <td>0.452595</td>\n",
       "      <td>1.981122</td>\n",
       "      <td>8.210248</td>\n",
       "      <td>4.195285</td>\n",
       "      <td>8.970920</td>\n",
       "      <td>...</td>\n",
       "      <td>7.716332</td>\n",
       "      <td>6.745802</td>\n",
       "      <td>8.602350</td>\n",
       "      <td>9.036654</td>\n",
       "      <td>10.336027</td>\n",
       "      <td>5.125213</td>\n",
       "      <td>5.401607</td>\n",
       "      <td>9.788359</td>\n",
       "      <td>10.090470</td>\n",
       "      <td>1.683023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.663618</td>\n",
       "      <td>4.507649</td>\n",
       "      <td>6.659068</td>\n",
       "      <td>10.196184</td>\n",
       "      <td>7.843375</td>\n",
       "      <td>0.434882</td>\n",
       "      <td>2.874246</td>\n",
       "      <td>8.306317</td>\n",
       "      <td>3.573556</td>\n",
       "      <td>8.524616</td>\n",
       "      <td>...</td>\n",
       "      <td>8.963286</td>\n",
       "      <td>7.744699</td>\n",
       "      <td>8.981473</td>\n",
       "      <td>8.665592</td>\n",
       "      <td>9.194823</td>\n",
       "      <td>6.076566</td>\n",
       "      <td>8.942805</td>\n",
       "      <td>9.694814</td>\n",
       "      <td>9.684365</td>\n",
       "      <td>3.292001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3.467853</td>\n",
       "      <td>3.581918</td>\n",
       "      <td>6.620243</td>\n",
       "      <td>9.706829</td>\n",
       "      <td>7.758510</td>\n",
       "      <td>0.515410</td>\n",
       "      <td>2.516797</td>\n",
       "      <td>6.842765</td>\n",
       "      <td>2.809661</td>\n",
       "      <td>7.663935</td>\n",
       "      <td>...</td>\n",
       "      <td>8.380816</td>\n",
       "      <td>7.792270</td>\n",
       "      <td>8.664963</td>\n",
       "      <td>8.344820</td>\n",
       "      <td>7.831035</td>\n",
       "      <td>5.726657</td>\n",
       "      <td>6.096154</td>\n",
       "      <td>9.244150</td>\n",
       "      <td>9.836473</td>\n",
       "      <td>5.355133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.854853</td>\n",
       "      <td>1.750478</td>\n",
       "      <td>7.226720</td>\n",
       "      <td>9.758691</td>\n",
       "      <td>5.952103</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.405856</td>\n",
       "      <td>7.373431</td>\n",
       "      <td>2.304861</td>\n",
       "      <td>8.922008</td>\n",
       "      <td>...</td>\n",
       "      <td>8.468738</td>\n",
       "      <td>8.466199</td>\n",
       "      <td>8.358093</td>\n",
       "      <td>7.422199</td>\n",
       "      <td>6.460507</td>\n",
       "      <td>5.297833</td>\n",
       "      <td>5.244290</td>\n",
       "      <td>9.987733</td>\n",
       "      <td>9.216872</td>\n",
       "      <td>6.551490</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 14041 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     gene_1    gene_2    gene_3     gene_4    gene_6   gene_11   gene_12  \\\n",
       "0  2.017209  3.265527  5.478487  10.431999  7.175175  1.334282  2.015391   \n",
       "2  3.511759  4.327199  6.881787   9.870730  6.972130  0.452595  1.981122   \n",
       "3  3.663618  4.507649  6.659068  10.196184  7.843375  0.434882  2.874246   \n",
       "5  3.467853  3.581918  6.620243   9.706829  7.758510  0.515410  2.516797   \n",
       "7  2.854853  1.750478  7.226720   9.758691  5.952103  0.000000  2.405856   \n",
       "\n",
       "    gene_19   gene_20   gene_22  ...  gene_20515  gene_20516  gene_20518  \\\n",
       "0  5.619994  1.334282  9.796088  ...    7.175175    9.249369    8.045563   \n",
       "2  8.210248  4.195285  8.970920  ...    7.716332    6.745802    8.602350   \n",
       "3  8.306317  3.573556  8.524616  ...    8.963286    7.744699    8.981473   \n",
       "5  6.842765  2.809661  7.663935  ...    8.380816    7.792270    8.664963   \n",
       "7  7.373431  2.304861  8.922008  ...    8.468738    8.466199    8.358093   \n",
       "\n",
       "   gene_20519  gene_20520  gene_20521  gene_20524  gene_20527  gene_20528  \\\n",
       "0    7.475709    7.205236    4.926711    7.220030    9.650743    8.921326   \n",
       "2    9.036654   10.336027    5.125213    5.401607    9.788359   10.090470   \n",
       "3    8.665592    9.194823    6.076566    8.942805    9.694814    9.684365   \n",
       "5    8.344820    7.831035    5.726657    6.096154    9.244150    9.836473   \n",
       "7    7.422199    6.460507    5.297833    5.244290    9.987733    9.216872   \n",
       "\n",
       "   gene_20529  \n",
       "0    5.286759  \n",
       "2    1.683023  \n",
       "3    3.292001  \n",
       "5    5.355133  \n",
       "7    6.551490  \n",
       "\n",
       "[5 rows x 14041 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the correlation matrix\n",
    "correlation_matrix_prad = df_prad.corr().abs()\n",
    "\n",
    "# Set the threshold for correlation\n",
    "threshold = 0.80\n",
    "\n",
    "# Find columns with correlation greater than or equal to the threshold\n",
    "high_corr_columns = np.where(correlation_matrix_prad >= threshold)\n",
    "\n",
    "# Create a set to store columns to remove\n",
    "columns_to_remove = set()\n",
    "\n",
    "# Iterate through the pairs of highly correlated columns\n",
    "for col1, col2 in zip(*high_corr_columns):\n",
    "    if col1 != col2 and col1 not in columns_to_remove:\n",
    "        columns_to_remove.add(col2)\n",
    "\n",
    "# Remove the identified columns\n",
    "df_prad_filtered = df_prad.drop(columns=df_prad.columns[list(columns_to_remove)],axis=1)\n",
    "df_prad_filtered.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_1</th>\n",
       "      <th>gene_2</th>\n",
       "      <th>gene_3</th>\n",
       "      <th>gene_4</th>\n",
       "      <th>gene_6</th>\n",
       "      <th>gene_11</th>\n",
       "      <th>gene_12</th>\n",
       "      <th>gene_19</th>\n",
       "      <th>gene_20</th>\n",
       "      <th>gene_22</th>\n",
       "      <th>...</th>\n",
       "      <th>gene_20518</th>\n",
       "      <th>gene_20519</th>\n",
       "      <th>gene_20521</th>\n",
       "      <th>gene_20522</th>\n",
       "      <th>gene_20523</th>\n",
       "      <th>gene_20524</th>\n",
       "      <th>gene_20525</th>\n",
       "      <th>gene_20526</th>\n",
       "      <th>gene_20527</th>\n",
       "      <th>gene_20529</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.592732</td>\n",
       "      <td>1.588421</td>\n",
       "      <td>7.586157</td>\n",
       "      <td>9.623011</td>\n",
       "      <td>6.816049</td>\n",
       "      <td>0.587845</td>\n",
       "      <td>2.466601</td>\n",
       "      <td>11.055208</td>\n",
       "      <td>3.562621</td>\n",
       "      <td>10.070470</td>\n",
       "      <td>...</td>\n",
       "      <td>9.458940</td>\n",
       "      <td>9.190867</td>\n",
       "      <td>4.593372</td>\n",
       "      <td>7.323865</td>\n",
       "      <td>9.740931</td>\n",
       "      <td>6.256586</td>\n",
       "      <td>8.381612</td>\n",
       "      <td>12.674552</td>\n",
       "      <td>10.517059</td>\n",
       "      <td>2.094168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3.460913</td>\n",
       "      <td>3.618474</td>\n",
       "      <td>5.661048</td>\n",
       "      <td>9.731217</td>\n",
       "      <td>8.435591</td>\n",
       "      <td>2.640644</td>\n",
       "      <td>2.719205</td>\n",
       "      <td>10.103576</td>\n",
       "      <td>5.679936</td>\n",
       "      <td>9.097097</td>\n",
       "      <td>...</td>\n",
       "      <td>7.498019</td>\n",
       "      <td>7.795481</td>\n",
       "      <td>5.350285</td>\n",
       "      <td>8.197320</td>\n",
       "      <td>9.907260</td>\n",
       "      <td>0.763921</td>\n",
       "      <td>10.088854</td>\n",
       "      <td>11.471137</td>\n",
       "      <td>9.768648</td>\n",
       "      <td>2.932118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1.131853</td>\n",
       "      <td>2.351515</td>\n",
       "      <td>6.845841</td>\n",
       "      <td>9.196860</td>\n",
       "      <td>7.853827</td>\n",
       "      <td>4.201469</td>\n",
       "      <td>2.564817</td>\n",
       "      <td>9.986724</td>\n",
       "      <td>5.412903</td>\n",
       "      <td>6.329574</td>\n",
       "      <td>...</td>\n",
       "      <td>9.222662</td>\n",
       "      <td>8.368066</td>\n",
       "      <td>5.235987</td>\n",
       "      <td>8.400832</td>\n",
       "      <td>9.524792</td>\n",
       "      <td>7.891079</td>\n",
       "      <td>10.579429</td>\n",
       "      <td>12.098697</td>\n",
       "      <td>10.550256</td>\n",
       "      <td>5.250397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>4.317702</td>\n",
       "      <td>3.642678</td>\n",
       "      <td>6.790003</td>\n",
       "      <td>8.435999</td>\n",
       "      <td>8.573628</td>\n",
       "      <td>1.757706</td>\n",
       "      <td>2.313043</td>\n",
       "      <td>9.708972</td>\n",
       "      <td>5.045229</td>\n",
       "      <td>8.758640</td>\n",
       "      <td>...</td>\n",
       "      <td>8.528849</td>\n",
       "      <td>7.771417</td>\n",
       "      <td>6.290371</td>\n",
       "      <td>8.951419</td>\n",
       "      <td>10.445253</td>\n",
       "      <td>1.757706</td>\n",
       "      <td>9.709429</td>\n",
       "      <td>12.364996</td>\n",
       "      <td>11.175213</td>\n",
       "      <td>4.912085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>4.416259</td>\n",
       "      <td>4.188978</td>\n",
       "      <td>6.664198</td>\n",
       "      <td>9.070819</td>\n",
       "      <td>7.915318</td>\n",
       "      <td>1.857026</td>\n",
       "      <td>2.168899</td>\n",
       "      <td>9.107863</td>\n",
       "      <td>4.568598</td>\n",
       "      <td>7.941206</td>\n",
       "      <td>...</td>\n",
       "      <td>8.777995</td>\n",
       "      <td>7.754193</td>\n",
       "      <td>5.786609</td>\n",
       "      <td>8.302470</td>\n",
       "      <td>10.431884</td>\n",
       "      <td>2.168899</td>\n",
       "      <td>9.402496</td>\n",
       "      <td>12.072283</td>\n",
       "      <td>10.637458</td>\n",
       "      <td>6.148450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 15678 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      gene_1    gene_2    gene_3    gene_4    gene_6   gene_11   gene_12  \\\n",
       "1   0.592732  1.588421  7.586157  9.623011  6.816049  0.587845  2.466601   \n",
       "16  3.460913  3.618474  5.661048  9.731217  8.435591  2.640644  2.719205   \n",
       "22  1.131853  2.351515  6.845841  9.196860  7.853827  4.201469  2.564817   \n",
       "25  4.317702  3.642678  6.790003  8.435999  8.573628  1.757706  2.313043   \n",
       "37  4.416259  4.188978  6.664198  9.070819  7.915318  1.857026  2.168899   \n",
       "\n",
       "      gene_19   gene_20    gene_22  ...  gene_20518  gene_20519  gene_20521  \\\n",
       "1   11.055208  3.562621  10.070470  ...    9.458940    9.190867    4.593372   \n",
       "16  10.103576  5.679936   9.097097  ...    7.498019    7.795481    5.350285   \n",
       "22   9.986724  5.412903   6.329574  ...    9.222662    8.368066    5.235987   \n",
       "25   9.708972  5.045229   8.758640  ...    8.528849    7.771417    6.290371   \n",
       "37   9.107863  4.568598   7.941206  ...    8.777995    7.754193    5.786609   \n",
       "\n",
       "    gene_20522  gene_20523  gene_20524  gene_20525  gene_20526  gene_20527  \\\n",
       "1     7.323865    9.740931    6.256586    8.381612   12.674552   10.517059   \n",
       "16    8.197320    9.907260    0.763921   10.088854   11.471137    9.768648   \n",
       "22    8.400832    9.524792    7.891079   10.579429   12.098697   10.550256   \n",
       "25    8.951419   10.445253    1.757706    9.709429   12.364996   11.175213   \n",
       "37    8.302470   10.431884    2.168899    9.402496   12.072283   10.637458   \n",
       "\n",
       "    gene_20529  \n",
       "1     2.094168  \n",
       "16    2.932118  \n",
       "22    5.250397  \n",
       "25    4.912085  \n",
       "37    6.148450  \n",
       "\n",
       "[5 rows x 15678 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the correlation matrix\n",
    "correlation_matrix_luad = df_luad.corr().abs()\n",
    "\n",
    "# Set the threshold for correlation\n",
    "threshold = 0.80\n",
    "\n",
    "# Find columns with correlation greater than or equal to the threshold\n",
    "high_corr_columns = np.where(correlation_matrix_luad >= threshold)\n",
    "\n",
    "# Create a set to store columns to remove\n",
    "columns_to_remove = set()\n",
    "\n",
    "# Iterate through the pairs of highly correlated columns\n",
    "for col1, col2 in zip(*high_corr_columns):\n",
    "    if col1 != col2 and col1 not in columns_to_remove:\n",
    "        columns_to_remove.add(col2)\n",
    "\n",
    "# Remove the identified columns\n",
    "df_luad_filtered = df_luad.drop(columns=df_luad.columns[list(columns_to_remove)],axis=1)\n",
    "df_luad_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_1</th>\n",
       "      <th>gene_2</th>\n",
       "      <th>gene_3</th>\n",
       "      <th>gene_4</th>\n",
       "      <th>gene_6</th>\n",
       "      <th>gene_11</th>\n",
       "      <th>gene_12</th>\n",
       "      <th>gene_19</th>\n",
       "      <th>gene_20</th>\n",
       "      <th>gene_22</th>\n",
       "      <th>...</th>\n",
       "      <th>gene_20520</th>\n",
       "      <th>gene_20521</th>\n",
       "      <th>gene_20522</th>\n",
       "      <th>gene_20523</th>\n",
       "      <th>gene_20524</th>\n",
       "      <th>gene_20525</th>\n",
       "      <th>gene_20526</th>\n",
       "      <th>gene_20527</th>\n",
       "      <th>gene_20528</th>\n",
       "      <th>gene_20529</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.655741</td>\n",
       "      <td>2.821547</td>\n",
       "      <td>6.539454</td>\n",
       "      <td>9.738265</td>\n",
       "      <td>6.566967</td>\n",
       "      <td>1.275841</td>\n",
       "      <td>2.141204</td>\n",
       "      <td>10.149150</td>\n",
       "      <td>2.967630</td>\n",
       "      <td>8.047238</td>\n",
       "      <td>...</td>\n",
       "      <td>9.298775</td>\n",
       "      <td>5.996032</td>\n",
       "      <td>8.891425</td>\n",
       "      <td>10.373790</td>\n",
       "      <td>7.181162</td>\n",
       "      <td>9.846910</td>\n",
       "      <td>11.922439</td>\n",
       "      <td>9.217749</td>\n",
       "      <td>9.461191</td>\n",
       "      <td>5.110372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3.992125</td>\n",
       "      <td>2.772730</td>\n",
       "      <td>6.546692</td>\n",
       "      <td>10.488252</td>\n",
       "      <td>7.690222</td>\n",
       "      <td>1.252839</td>\n",
       "      <td>2.579977</td>\n",
       "      <td>10.147625</td>\n",
       "      <td>4.287908</td>\n",
       "      <td>8.343976</td>\n",
       "      <td>...</td>\n",
       "      <td>10.461326</td>\n",
       "      <td>6.721974</td>\n",
       "      <td>9.597533</td>\n",
       "      <td>9.763753</td>\n",
       "      <td>7.933278</td>\n",
       "      <td>10.952880</td>\n",
       "      <td>12.498919</td>\n",
       "      <td>10.389954</td>\n",
       "      <td>10.390255</td>\n",
       "      <td>7.828321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3.492071</td>\n",
       "      <td>3.553373</td>\n",
       "      <td>7.151707</td>\n",
       "      <td>10.253446</td>\n",
       "      <td>8.301258</td>\n",
       "      <td>1.999567</td>\n",
       "      <td>3.381962</td>\n",
       "      <td>7.965224</td>\n",
       "      <td>2.146395</td>\n",
       "      <td>8.115408</td>\n",
       "      <td>...</td>\n",
       "      <td>9.404882</td>\n",
       "      <td>6.231006</td>\n",
       "      <td>9.061674</td>\n",
       "      <td>10.042261</td>\n",
       "      <td>6.580103</td>\n",
       "      <td>9.785004</td>\n",
       "      <td>12.383834</td>\n",
       "      <td>9.869359</td>\n",
       "      <td>8.991768</td>\n",
       "      <td>5.735121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.551048</td>\n",
       "      <td>3.529846</td>\n",
       "      <td>6.326825</td>\n",
       "      <td>10.633849</td>\n",
       "      <td>8.944659</td>\n",
       "      <td>0.501821</td>\n",
       "      <td>2.479282</td>\n",
       "      <td>8.526534</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.717848</td>\n",
       "      <td>...</td>\n",
       "      <td>11.119337</td>\n",
       "      <td>3.920732</td>\n",
       "      <td>7.846901</td>\n",
       "      <td>9.379835</td>\n",
       "      <td>7.195800</td>\n",
       "      <td>8.907645</td>\n",
       "      <td>12.079481</td>\n",
       "      <td>9.218936</td>\n",
       "      <td>8.104101</td>\n",
       "      <td>3.660894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.964842</td>\n",
       "      <td>2.183010</td>\n",
       "      <td>6.596832</td>\n",
       "      <td>10.248141</td>\n",
       "      <td>7.087251</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.896214</td>\n",
       "      <td>7.993499</td>\n",
       "      <td>2.750306</td>\n",
       "      <td>9.580091</td>\n",
       "      <td>...</td>\n",
       "      <td>10.266295</td>\n",
       "      <td>6.078847</td>\n",
       "      <td>10.829588</td>\n",
       "      <td>10.404567</td>\n",
       "      <td>6.588536</td>\n",
       "      <td>8.952596</td>\n",
       "      <td>10.461725</td>\n",
       "      <td>10.582857</td>\n",
       "      <td>9.488141</td>\n",
       "      <td>3.261967</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 15394 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      gene_1    gene_2    gene_3     gene_4    gene_6   gene_11   gene_12  \\\n",
       "4   2.655741  2.821547  6.539454   9.738265  6.566967  1.275841  2.141204   \n",
       "8   3.992125  2.772730  6.546692  10.488252  7.690222  1.252839  2.579977   \n",
       "10  3.492071  3.553373  7.151707  10.253446  8.301258  1.999567  3.381962   \n",
       "13  1.551048  3.529846  6.326825  10.633849  8.944659  0.501821  2.479282   \n",
       "14  1.964842  2.183010  6.596832  10.248141  7.087251  0.000000  2.896214   \n",
       "\n",
       "      gene_19   gene_20    gene_22  ...  gene_20520  gene_20521  gene_20522  \\\n",
       "4   10.149150  2.967630   8.047238  ...    9.298775    5.996032    8.891425   \n",
       "8   10.147625  4.287908   8.343976  ...   10.461326    6.721974    9.597533   \n",
       "10   7.965224  2.146395   8.115408  ...    9.404882    6.231006    9.061674   \n",
       "13   8.526534  0.000000  10.717848  ...   11.119337    3.920732    7.846901   \n",
       "14   7.993499  2.750306   9.580091  ...   10.266295    6.078847   10.829588   \n",
       "\n",
       "    gene_20523  gene_20524  gene_20525  gene_20526  gene_20527  gene_20528  \\\n",
       "4    10.373790    7.181162    9.846910   11.922439    9.217749    9.461191   \n",
       "8     9.763753    7.933278   10.952880   12.498919   10.389954   10.390255   \n",
       "10   10.042261    6.580103    9.785004   12.383834    9.869359    8.991768   \n",
       "13    9.379835    7.195800    8.907645   12.079481    9.218936    8.104101   \n",
       "14   10.404567    6.588536    8.952596   10.461725   10.582857    9.488141   \n",
       "\n",
       "    gene_20529  \n",
       "4     5.110372  \n",
       "8     7.828321  \n",
       "10    5.735121  \n",
       "13    3.660894  \n",
       "14    3.261967  \n",
       "\n",
       "[5 rows x 15394 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the correlation matrix\n",
    "correlation_matrix_brca = df_brca.corr().abs()\n",
    "\n",
    "# Set the threshold for correlation\n",
    "threshold = 0.80\n",
    "\n",
    "# Find columns with correlation greater than or equal to the threshold\n",
    "high_corr_columns = np.where(correlation_matrix_brca >= threshold)\n",
    "\n",
    "# Create a set to store columns to remove\n",
    "columns_to_remove = set()\n",
    "\n",
    "# Iterate through the pairs of highly correlated columns\n",
    "for col1, col2 in zip(*high_corr_columns):\n",
    "    if col1 != col2 and col1 not in columns_to_remove:\n",
    "        columns_to_remove.add(col2)\n",
    "\n",
    "# Remove the identified columns\n",
    "df_brca_filtered = df_brca.drop(columns=df_brca.columns[list(columns_to_remove)],axis=1)\n",
    "df_brca_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_1</th>\n",
       "      <th>gene_2</th>\n",
       "      <th>gene_3</th>\n",
       "      <th>gene_4</th>\n",
       "      <th>gene_6</th>\n",
       "      <th>gene_11</th>\n",
       "      <th>gene_12</th>\n",
       "      <th>gene_19</th>\n",
       "      <th>gene_20</th>\n",
       "      <th>gene_22</th>\n",
       "      <th>...</th>\n",
       "      <th>gene_20517</th>\n",
       "      <th>gene_20518</th>\n",
       "      <th>gene_20519</th>\n",
       "      <th>gene_20522</th>\n",
       "      <th>gene_20523</th>\n",
       "      <th>gene_20524</th>\n",
       "      <th>gene_20526</th>\n",
       "      <th>gene_20527</th>\n",
       "      <th>gene_20528</th>\n",
       "      <th>gene_20529</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.224966</td>\n",
       "      <td>1.691177</td>\n",
       "      <td>6.572007</td>\n",
       "      <td>9.640511</td>\n",
       "      <td>6.754888</td>\n",
       "      <td>1.476796</td>\n",
       "      <td>3.023841</td>\n",
       "      <td>7.424200</td>\n",
       "      <td>1.224966</td>\n",
       "      <td>9.973640</td>\n",
       "      <td>...</td>\n",
       "      <td>6.724787</td>\n",
       "      <td>7.507858</td>\n",
       "      <td>6.862712</td>\n",
       "      <td>7.927968</td>\n",
       "      <td>9.673966</td>\n",
       "      <td>1.877744</td>\n",
       "      <td>13.256060</td>\n",
       "      <td>9.664486</td>\n",
       "      <td>9.244219</td>\n",
       "      <td>8.330912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.941181</td>\n",
       "      <td>2.663276</td>\n",
       "      <td>6.561690</td>\n",
       "      <td>9.376293</td>\n",
       "      <td>7.860323</td>\n",
       "      <td>1.021409</td>\n",
       "      <td>3.153092</td>\n",
       "      <td>9.796182</td>\n",
       "      <td>4.848317</td>\n",
       "      <td>8.614732</td>\n",
       "      <td>...</td>\n",
       "      <td>7.222080</td>\n",
       "      <td>8.198813</td>\n",
       "      <td>7.234137</td>\n",
       "      <td>8.395474</td>\n",
       "      <td>10.405099</td>\n",
       "      <td>5.070922</td>\n",
       "      <td>11.749802</td>\n",
       "      <td>10.397407</td>\n",
       "      <td>9.300561</td>\n",
       "      <td>5.276564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3.004519</td>\n",
       "      <td>3.007178</td>\n",
       "      <td>6.524205</td>\n",
       "      <td>9.062661</td>\n",
       "      <td>7.995937</td>\n",
       "      <td>1.077516</td>\n",
       "      <td>2.233060</td>\n",
       "      <td>8.317150</td>\n",
       "      <td>3.802669</td>\n",
       "      <td>7.697197</td>\n",
       "      <td>...</td>\n",
       "      <td>7.119553</td>\n",
       "      <td>8.346975</td>\n",
       "      <td>7.298860</td>\n",
       "      <td>8.702713</td>\n",
       "      <td>10.392532</td>\n",
       "      <td>2.711935</td>\n",
       "      <td>11.623369</td>\n",
       "      <td>10.327115</td>\n",
       "      <td>9.772328</td>\n",
       "      <td>3.597400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1.541465</td>\n",
       "      <td>2.541540</td>\n",
       "      <td>6.843255</td>\n",
       "      <td>9.444468</td>\n",
       "      <td>5.479091</td>\n",
       "      <td>0.401194</td>\n",
       "      <td>2.456438</td>\n",
       "      <td>10.602717</td>\n",
       "      <td>1.190615</td>\n",
       "      <td>8.026241</td>\n",
       "      <td>...</td>\n",
       "      <td>7.028227</td>\n",
       "      <td>8.188826</td>\n",
       "      <td>7.103917</td>\n",
       "      <td>8.619974</td>\n",
       "      <td>10.088165</td>\n",
       "      <td>2.178492</td>\n",
       "      <td>12.380688</td>\n",
       "      <td>10.741298</td>\n",
       "      <td>9.808642</td>\n",
       "      <td>5.777772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>3.529783</td>\n",
       "      <td>2.976712</td>\n",
       "      <td>6.038416</td>\n",
       "      <td>9.437973</td>\n",
       "      <td>5.893828</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.250537</td>\n",
       "      <td>10.199182</td>\n",
       "      <td>1.732443</td>\n",
       "      <td>8.697826</td>\n",
       "      <td>...</td>\n",
       "      <td>7.980968</td>\n",
       "      <td>9.228617</td>\n",
       "      <td>7.910757</td>\n",
       "      <td>9.186099</td>\n",
       "      <td>10.241781</td>\n",
       "      <td>1.349705</td>\n",
       "      <td>12.826908</td>\n",
       "      <td>9.538280</td>\n",
       "      <td>10.131587</td>\n",
       "      <td>7.738390</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 14880 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      gene_1    gene_2    gene_3    gene_4    gene_6   gene_11   gene_12  \\\n",
       "6   1.224966  1.691177  6.572007  9.640511  6.754888  1.476796  3.023841   \n",
       "11  2.941181  2.663276  6.561690  9.376293  7.860323  1.021409  3.153092   \n",
       "17  3.004519  3.007178  6.524205  9.062661  7.995937  1.077516  2.233060   \n",
       "18  1.541465  2.541540  6.843255  9.444468  5.479091  0.401194  2.456438   \n",
       "21  3.529783  2.976712  6.038416  9.437973  5.893828  0.000000  3.250537   \n",
       "\n",
       "      gene_19   gene_20   gene_22  ...  gene_20517  gene_20518  gene_20519  \\\n",
       "6    7.424200  1.224966  9.973640  ...    6.724787    7.507858    6.862712   \n",
       "11   9.796182  4.848317  8.614732  ...    7.222080    8.198813    7.234137   \n",
       "17   8.317150  3.802669  7.697197  ...    7.119553    8.346975    7.298860   \n",
       "18  10.602717  1.190615  8.026241  ...    7.028227    8.188826    7.103917   \n",
       "21  10.199182  1.732443  8.697826  ...    7.980968    9.228617    7.910757   \n",
       "\n",
       "    gene_20522  gene_20523  gene_20524  gene_20526  gene_20527  gene_20528  \\\n",
       "6     7.927968    9.673966    1.877744   13.256060    9.664486    9.244219   \n",
       "11    8.395474   10.405099    5.070922   11.749802   10.397407    9.300561   \n",
       "17    8.702713   10.392532    2.711935   11.623369   10.327115    9.772328   \n",
       "18    8.619974   10.088165    2.178492   12.380688   10.741298    9.808642   \n",
       "21    9.186099   10.241781    1.349705   12.826908    9.538280   10.131587   \n",
       "\n",
       "    gene_20529  \n",
       "6     8.330912  \n",
       "11    5.276564  \n",
       "17    3.597400  \n",
       "18    5.777772  \n",
       "21    7.738390  \n",
       "\n",
       "[5 rows x 14880 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the correlation matrix\n",
    "correlation_matrix_kirc = df_kirc.corr().abs()\n",
    "\n",
    "# Set the threshold for correlation\n",
    "threshold = 0.80\n",
    "\n",
    "# Find columns with correlation greater than or equal to the threshold\n",
    "high_corr_columns = np.where(correlation_matrix_kirc >= threshold)\n",
    "\n",
    "# Create a set to store columns to remove\n",
    "columns_to_remove = set()\n",
    "\n",
    "# Iterate through the pairs of highly correlated columns\n",
    "for col1, col2 in zip(*high_corr_columns):\n",
    "    if col1 != col2 and col1 not in columns_to_remove:\n",
    "        columns_to_remove.add(col2)\n",
    "\n",
    "# Remove the identified columns\n",
    "df_kirc_filtered = df_kirc.drop(columns=df_kirc.columns[list(columns_to_remove)],axis=1)\n",
    "df_kirc_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_1</th>\n",
       "      <th>gene_2</th>\n",
       "      <th>gene_3</th>\n",
       "      <th>gene_4</th>\n",
       "      <th>gene_6</th>\n",
       "      <th>gene_11</th>\n",
       "      <th>gene_12</th>\n",
       "      <th>gene_19</th>\n",
       "      <th>gene_20</th>\n",
       "      <th>gene_22</th>\n",
       "      <th>...</th>\n",
       "      <th>gene_20518</th>\n",
       "      <th>gene_20521</th>\n",
       "      <th>gene_20522</th>\n",
       "      <th>gene_20523</th>\n",
       "      <th>gene_20524</th>\n",
       "      <th>gene_20525</th>\n",
       "      <th>gene_20526</th>\n",
       "      <th>gene_20527</th>\n",
       "      <th>gene_20528</th>\n",
       "      <th>gene_20529</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2.325242</td>\n",
       "      <td>3.247092</td>\n",
       "      <td>8.174007</td>\n",
       "      <td>10.065053</td>\n",
       "      <td>7.487446</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.759582</td>\n",
       "      <td>6.944554</td>\n",
       "      <td>2.951215</td>\n",
       "      <td>7.163660</td>\n",
       "      <td>...</td>\n",
       "      <td>8.745136</td>\n",
       "      <td>5.765299</td>\n",
       "      <td>9.582618</td>\n",
       "      <td>10.097084</td>\n",
       "      <td>0.566572</td>\n",
       "      <td>9.414850</td>\n",
       "      <td>12.513264</td>\n",
       "      <td>9.714840</td>\n",
       "      <td>9.413833</td>\n",
       "      <td>4.470146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>3.446190</td>\n",
       "      <td>3.620962</td>\n",
       "      <td>7.171917</td>\n",
       "      <td>9.796559</td>\n",
       "      <td>8.175245</td>\n",
       "      <td>1.972509</td>\n",
       "      <td>2.218967</td>\n",
       "      <td>8.617963</td>\n",
       "      <td>3.823780</td>\n",
       "      <td>7.783575</td>\n",
       "      <td>...</td>\n",
       "      <td>8.647196</td>\n",
       "      <td>5.019635</td>\n",
       "      <td>8.453184</td>\n",
       "      <td>10.082854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.389552</td>\n",
       "      <td>12.123349</td>\n",
       "      <td>11.552194</td>\n",
       "      <td>9.052753</td>\n",
       "      <td>1.675093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>2.554196</td>\n",
       "      <td>1.296134</td>\n",
       "      <td>8.770277</td>\n",
       "      <td>10.462901</td>\n",
       "      <td>8.013993</td>\n",
       "      <td>0.736562</td>\n",
       "      <td>3.543446</td>\n",
       "      <td>7.243802</td>\n",
       "      <td>3.221537</td>\n",
       "      <td>10.231857</td>\n",
       "      <td>...</td>\n",
       "      <td>7.985831</td>\n",
       "      <td>4.873543</td>\n",
       "      <td>8.185569</td>\n",
       "      <td>9.876580</td>\n",
       "      <td>1.736302</td>\n",
       "      <td>9.230261</td>\n",
       "      <td>12.363812</td>\n",
       "      <td>9.441293</td>\n",
       "      <td>9.639285</td>\n",
       "      <td>2.221630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>3.667699</td>\n",
       "      <td>3.083179</td>\n",
       "      <td>7.997292</td>\n",
       "      <td>9.516886</td>\n",
       "      <td>7.925424</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.762157</td>\n",
       "      <td>10.178167</td>\n",
       "      <td>2.797719</td>\n",
       "      <td>7.065637</td>\n",
       "      <td>...</td>\n",
       "      <td>9.017591</td>\n",
       "      <td>6.783260</td>\n",
       "      <td>9.257888</td>\n",
       "      <td>10.331936</td>\n",
       "      <td>0.732486</td>\n",
       "      <td>8.922457</td>\n",
       "      <td>12.182373</td>\n",
       "      <td>10.525648</td>\n",
       "      <td>9.444771</td>\n",
       "      <td>3.449205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>3.821935</td>\n",
       "      <td>3.717605</td>\n",
       "      <td>7.593100</td>\n",
       "      <td>10.205964</td>\n",
       "      <td>7.613185</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.982364</td>\n",
       "      <td>9.138154</td>\n",
       "      <td>2.787182</td>\n",
       "      <td>7.445346</td>\n",
       "      <td>...</td>\n",
       "      <td>8.900453</td>\n",
       "      <td>6.208320</td>\n",
       "      <td>9.137073</td>\n",
       "      <td>10.057464</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.988114</td>\n",
       "      <td>12.993375</td>\n",
       "      <td>10.565978</td>\n",
       "      <td>9.184890</td>\n",
       "      <td>3.102541</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 15012 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      gene_1    gene_2    gene_3     gene_4    gene_6   gene_11   gene_12  \\\n",
       "26  2.325242  3.247092  8.174007  10.065053  7.487446  0.000000  2.759582   \n",
       "47  3.446190  3.620962  7.171917   9.796559  8.175245  1.972509  2.218967   \n",
       "54  2.554196  1.296134  8.770277  10.462901  8.013993  0.736562  3.543446   \n",
       "57  3.667699  3.083179  7.997292   9.516886  7.925424  0.000000  3.762157   \n",
       "65  3.821935  3.717605  7.593100  10.205964  7.613185  0.000000  1.982364   \n",
       "\n",
       "      gene_19   gene_20    gene_22  ...  gene_20518  gene_20521  gene_20522  \\\n",
       "26   6.944554  2.951215   7.163660  ...    8.745136    5.765299    9.582618   \n",
       "47   8.617963  3.823780   7.783575  ...    8.647196    5.019635    8.453184   \n",
       "54   7.243802  3.221537  10.231857  ...    7.985831    4.873543    8.185569   \n",
       "57  10.178167  2.797719   7.065637  ...    9.017591    6.783260    9.257888   \n",
       "65   9.138154  2.787182   7.445346  ...    8.900453    6.208320    9.137073   \n",
       "\n",
       "    gene_20523  gene_20524  gene_20525  gene_20526  gene_20527  gene_20528  \\\n",
       "26   10.097084    0.566572    9.414850   12.513264    9.714840    9.413833   \n",
       "47   10.082854    0.000000    9.389552   12.123349   11.552194    9.052753   \n",
       "54    9.876580    1.736302    9.230261   12.363812    9.441293    9.639285   \n",
       "57   10.331936    0.732486    8.922457   12.182373   10.525648    9.444771   \n",
       "65   10.057464    0.000000    8.988114   12.993375   10.565978    9.184890   \n",
       "\n",
       "    gene_20529  \n",
       "26    4.470146  \n",
       "47    1.675093  \n",
       "54    2.221630  \n",
       "57    3.449205  \n",
       "65    3.102541  \n",
       "\n",
       "[5 rows x 15012 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the correlation matrix\n",
    "correlation_matrix_coad = df_coad.corr().abs()\n",
    "\n",
    "# Set the threshold for correlation\n",
    "threshold = 0.80\n",
    "\n",
    "# Find columns with correlation greater than or equal to the threshold\n",
    "high_corr_columns = np.where(correlation_matrix_coad >= threshold)\n",
    "\n",
    "# Create a set to store columns to remove\n",
    "columns_to_remove = set()\n",
    "\n",
    "# Iterate through the pairs of highly correlated columns\n",
    "for col1, col2 in zip(*high_corr_columns):\n",
    "    if col1 != col2 and col1 not in columns_to_remove:\n",
    "        columns_to_remove.add(col2)\n",
    "\n",
    "# Remove the identified columns\n",
    "df_coad_filtered = df_coad.drop(columns=df_coad.columns[list(columns_to_remove)],axis=1)\n",
    "df_coad_filtered.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate intra distance for each cancer type BRCA, PRAD, LUAD, COAD, KIRC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(157.93830753356286)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Calculate mahalanobis distance for intra class PRAD\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_prad = df_prad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "df_prad_filtered_mean = df_sampled_prad.mean()\n",
    "\n",
    "df_prad_filtered_cov = df_sampled_prad.cov()\n",
    "\n",
    "df_prad_filtered_cov_inv = np.linalg.inv(df_prad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "for index in  range(0, df_sampled_prad.shape[0]):\n",
    "    diff = df_sampled_prad.iloc[index, :].values - df_prad_filtered_mean.values\n",
    "    distance = np.dot(diff.T, np.dot(df_prad_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "max_intra_dist_prad = np.max(distances)\n",
    "\n",
    "max_intra_dist_prad\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(126.62846541864772)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Calculate mahalanobis distance for intra class LUAD\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_luad = df_luad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "df_luad_filtered_mean = df_sampled_luad.mean()\n",
    "\n",
    "df_luad_filtered_cov = df_sampled_luad.cov()\n",
    "\n",
    "df_luad_filtered_cov_inv = np.linalg.inv(df_luad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "for index in  range(0, df_sampled_luad.shape[0]):\n",
    "    diff = df_sampled_luad.iloc[index, :].values - df_luad_filtered_mean.values\n",
    "    distance = np.dot(diff.T, np.dot(df_luad_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "max_intra_dist_luad = np.max(distances)\n",
    "\n",
    "max_intra_dist_luad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(177.08547261187474)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Calculate mahalanobis distance for intra class BRCA\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_brca = df_brca_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "df_brca_filtered_mean = df_sampled_brca.mean()\n",
    "\n",
    "df_brca_filtered_cov = df_sampled_brca.cov()\n",
    "\n",
    "df_brca_filtered_cov_inv = np.linalg.inv(df_brca_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "for index in  range(0, df_sampled_brca.shape[0]):\n",
    "    diff = df_sampled_brca.iloc[index, :].values - df_brca_filtered_mean.values\n",
    "    distance = np.dot(diff.T, np.dot(df_brca_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "    \n",
    "max_intra_dist_brca = np.max(distances)\n",
    "\n",
    "max_intra_dist_brca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(85.46626228498594)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Calculate mahalanobis distance for intra class COAD\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_coad = df_coad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "df_coad_filtered_mean = df_sampled_coad.mean()\n",
    "\n",
    "df_coad_filtered_cov = df_sampled_coad.cov()\n",
    "\n",
    "df_coad_filtered_cov_inv = np.linalg.inv(df_coad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "for index in range(0, df_sampled_coad.shape[0]):\n",
    "    diff = df_sampled_coad.iloc[index, :].values - df_coad_filtered_mean.values\n",
    "    distance = np.dot(diff.T, np.dot(df_coad_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "    \n",
    "max_intra_dist_coad = np.max(distances)\n",
    "\n",
    "max_intra_dist_coad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(173.3680122813765)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Calculate mahalanobis distance for intra class KIRC\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_kirc = df_kirc_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "df_kirc_filtered_mean = df_sampled_kirc.mean()\n",
    "\n",
    "df_kirc_filtered_cov = df_sampled_kirc.cov()\n",
    "\n",
    "df_kirc_filtered_cov_inv = np.linalg.inv(df_kirc_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "for index in  range(0, df_sampled_kirc.shape[0]):\n",
    "    diff = df_sampled_kirc.iloc[index, :].values - df_kirc_filtered_mean.values\n",
    "    distance = np.dot(diff.T, np.dot(df_kirc_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "max_intra_dist_kirc = np.max(distances)\n",
    "\n",
    "max_intra_dist_kirc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate the overlap for each mahalanobis distance for each cancer type : PRAD-LUAD, PRAD-COAD, PRAD-KIRC, PRAD-BRCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(4.041765435886695e-08)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_luad = df_luad_filtered.sample(axis=1, n=14000)\n",
    "df_sampled_prad = df_prad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "\n",
    "df_luad_filtered_mean = df_sampled_luad.mean()\n",
    "df_prad_filtered_mean = df_sampled_prad.mean()\n",
    "\n",
    "df_luad_filtered_cov = df_sampled_luad.cov()\n",
    "df_prad_filtered_cov = df_sampled_prad.cov()\n",
    "\n",
    "df_luad_filtered_cov_inv = np.linalg.inv(df_luad_filtered_cov)\n",
    "df_prad_filtered_cov_inv = np.linalg.inv(df_prad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_prad_mean and luad_gene\n",
    "for index in  range(0, df_sampled_luad.shape[0]):\n",
    "    diff = df_sampled_luad.iloc[index, :].values - df_prad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_luad_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "    \n",
    "minimum_inter_prad_luad = np.min(distances)\n",
    "\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_luad_mean and prad_gene\n",
    "for index in  range(0, df_sampled_prad.shape[0]):\n",
    "    diff = df_sampled_prad.iloc[index, :].values - df_luad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_prad_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_luad_prad = np.min(distances)\n",
    "\n",
    "minimum_inter =  min(minimum_inter_prad_luad,minimum_inter_luad_prad)\n",
    "overlap_prad_luad = (max_intra_dist_luad + max_intra_dist_prad)/(2*minimum_inter)\n",
    "overlap_prad_luad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.03721714401816614)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#PRAD-COAD\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_coad = df_coad_filtered.sample(axis=1, n=14000)\n",
    "df_sampled_prad = df_prad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "\n",
    "df_coad_filtered_mean = df_sampled_coad.mean()\n",
    "df_prad_filtered_mean = df_sampled_prad.mean()\n",
    "\n",
    "df_coad_filtered_cov = df_sampled_coad.cov()\n",
    "df_prad_filtered_cov = df_sampled_prad.cov()\n",
    "\n",
    "df_coad_filtered_cov_inv = np.linalg.inv(df_coad_filtered_cov)\n",
    "df_prad_filtered_cov_inv = np.linalg.inv(df_prad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_prad_mean and coad_gene\n",
    "for index in  range(0, df_sampled_coad.shape[0]):\n",
    "    diff = df_sampled_coad.iloc[index, :].values - df_prad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_coad_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "    \n",
    "minimum_inter_prad_coad = np.min(distances)\n",
    "\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_coad_mean and prad_gene\n",
    "for index in  range(0, df_sampled_prad.shape[0]):\n",
    "    diff = df_sampled_prad.iloc[index, :].values - df_coad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_prad_filtered_cov, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_coad_prad = np.min(distances)\n",
    "\n",
    "minimum_inter =  min(minimum_inter_coad_prad,minimum_inter_prad_coad)\n",
    "overlap_prad_coad = (max_intra_dist_coad+max_intra_dist_prad)/(2*minimum_inter)\n",
    "overlap_prad_coad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.048083581788951536)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#PRAD-KIRC\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_kirc = df_kirc_filtered.sample(axis=1, n=14000)\n",
    "df_sampled_prad = df_prad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "\n",
    "df_kirc_filtered_mean = df_sampled_kirc.mean()\n",
    "df_prad_filtered_mean = df_sampled_prad.mean()\n",
    "\n",
    "df_kirc_filtered_cov = df_sampled_kirc.cov()\n",
    "df_prad_filtered_cov = df_sampled_prad.cov()\n",
    "\n",
    "df_kirc_filtered_cov_inv = np.linalg.inv(df_kirc_filtered_cov)\n",
    "df_prad_filtered_cov_inv = np.linalg.inv(df_prad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_prad_mean and kirc_gene\n",
    "for index in  range(0, df_sampled_kirc.shape[0]):\n",
    "    diff = df_sampled_kirc.iloc[index, :].values - df_prad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_kirc_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_prad_kirc = np.min(distances)\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_kirc_mean and prad_gene\n",
    "for index in  range(0, df_sampled_prad.shape[0]):\n",
    "    diff = df_sampled_prad.iloc[index, :].values - df_kirc_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_prad_filtered_cov, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "    \n",
    "minimum_inter_kirc_prad = np.min(distances)\n",
    "\n",
    "\n",
    "minimum_inter =  min(minimum_inter_kirc_prad,minimum_inter_prad_kirc)\n",
    "overlap_prad_kirc = (max_intra_dist_kirc+max_intra_dist_prad)/(2*minimum_inter)\n",
    "overlap_prad_kirc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.05196358188848537)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#PRAD-BRCA\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_brca = df_brca_filtered.sample(axis=1, n=14000)\n",
    "df_sampled_prad = df_prad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "\n",
    "df_brca_filtered_mean = df_sampled_brca.mean()\n",
    "df_prad_filtered_mean = df_sampled_prad.mean()\n",
    "\n",
    "df_brca_filtered_cov = df_sampled_brca.cov()\n",
    "df_prad_filtered_cov = df_sampled_prad.cov()\n",
    "\n",
    "df_brca_filtered_cov_inv = np.linalg.inv(df_brca_filtered_cov)\n",
    "df_prad_filtered_cov_inv = np.linalg.inv(df_prad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "#Take the center of df_prad_mean and brca_gene\n",
    "for index in  range(0, df_sampled_brca.shape[0]):\n",
    "    diff = df_sampled_brca.iloc[index, :].values - df_prad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_brca_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_prad_brca = np.min(distances)\n",
    "\n",
    "\n",
    "distances = []\n",
    "#Take the center of df_brca_mean and prad_gene\n",
    "for index in  range(0, df_sampled_prad.shape[0]):\n",
    "    diff = df_sampled_prad.iloc[index, :].values - df_brca_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_prad_filtered_cov, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_brca_prad = np.min(distances)\n",
    "\n",
    "minimum_inter =  min(minimum_inter_brca_prad,minimum_inter_prad_brca)\n",
    "overlap_prad_brca = (max_intra_dist_prad+max_intra_dist_brca)/(2*minimum_inter)\n",
    "overlap_prad_brca"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate the overlap for each mahalanobis distance for each cancer type : COAD-BRCA,COAD-LUAD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.07598872328536402)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#COAD-BRCA\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_brca = df_brca_filtered.sample(axis=1, n=14000)\n",
    "df_sampled_coad = df_coad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "\n",
    "df_brca_filtered_mean = df_sampled_brca.mean()\n",
    "df_coad_filtered_mean = df_sampled_coad.mean()\n",
    "\n",
    "df_brca_filtered_cov = df_sampled_brca.cov()\n",
    "df_coad_filtered_cov = df_sampled_coad.cov()\n",
    "\n",
    "df_brca_filtered_cov_inv = np.linalg.inv(df_brca_filtered_cov)\n",
    "df_coad_filtered_cov_inv = np.linalg.inv(df_coad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_coad_mean and brca_gene\n",
    "for index in  range(0, df_sampled_brca.shape[0]):\n",
    "    diff = df_sampled_brca.iloc[index, :].values - df_coad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_brca_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_coad_brca = np.min(distances)\n",
    "\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_brca_mean and coad_gene\n",
    "for index in range(0, df_sampled_coad.shape[0]):\n",
    "    diff = df_sampled_coad.iloc[index, :].values - df_brca_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_coad_filtered_cov, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_brca_coad = np.min(distances)\n",
    "\n",
    "minimum_inter =  min(minimum_inter_brca_coad,minimum_inter_coad_brca)\n",
    "overlap_coad_brca = (max_intra_dist_coad+max_intra_dist_brca)/(2*minimum_inter)\n",
    "overlap_coad_brca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.05564161684979325)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#COAD-LUAD\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_luad = df_luad_filtered.sample(axis=1, n=14000)\n",
    "df_sampled_coad = df_coad_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "\n",
    "df_luad_filtered_mean = df_sampled_luad.mean()\n",
    "df_coad_filtered_mean = df_sampled_coad.mean()\n",
    "\n",
    "df_luad_filtered_cov = df_sampled_luad.cov()\n",
    "df_coad_filtered_cov = df_sampled_coad.cov()\n",
    "\n",
    "df_luad_filtered_cov_inv = np.linalg.inv(df_luad_filtered_cov)\n",
    "df_coad_filtered_cov_inv = np.linalg.inv(df_coad_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_coad_mean and luad_gene\n",
    "for index in  range(0, df_sampled_luad.shape[0]):\n",
    "    diff = df_sampled_luad.iloc[index, :].values - df_coad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_luad_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "    \n",
    "minimum_inter_coad_luad = np.min(distances)\n",
    "\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_luad_mean and coad_gene\n",
    "for index in range(0, df_sampled_coad.shape[0]):\n",
    "    diff = df_sampled_coad.iloc[index, :].values - df_luad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_coad_filtered_cov, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_luad_coad = np.min(distances)\n",
    "\n",
    "\n",
    "minimum_inter =  min(minimum_inter_luad_coad,minimum_inter_coad_luad)\n",
    "overlap_coad_luad = (max_intra_dist_coad+max_intra_dist_luad)/(2*minimum_inter)\n",
    "overlap_coad_luad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the overlap for each mahalanobis distance for each cancer type : LUAD-BRCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.043930155683510386)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#LUAD-BRCA\n",
    "np.random.seed(1)\n",
    "\n",
    "df_sampled_luad = df_luad_filtered.sample(axis=1, n=14000)\n",
    "df_sampled_brca = df_brca_filtered.sample(axis=1, n=14000)\n",
    "\n",
    "\n",
    "df_luad_filtered_mean = df_sampled_luad.mean()\n",
    "df_brca_filtered_mean = df_sampled_brca.mean()\n",
    "\n",
    "df_luad_filtered_cov = df_sampled_luad.cov()\n",
    "df_brca_filtered_cov = df_sampled_brca.cov()\n",
    "\n",
    "df_luad_filtered_cov_inv = np.linalg.inv(df_luad_filtered_cov)\n",
    "df_brca_filtered_cov_inv = np.linalg.inv(df_brca_filtered_cov)\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_brca_mean and luad_gene\n",
    "for index in  range(0, df_sampled_luad.shape[0]):\n",
    "    diff = df_sampled_luad.iloc[index, :].values - df_brca_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_luad_filtered_cov_inv, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "    \n",
    "minimum_inter_brca_luad = np.min(distances)\n",
    "\n",
    "\n",
    "distances = []\n",
    "\n",
    "#Take the center of df_luad_mean and brca_gene\n",
    "for index in range(0, df_sampled_brca.shape[0]):\n",
    "    diff = df_sampled_brca.iloc[index, :].values - df_luad_filtered_mean.values\n",
    "    distance = np.dot(diff, np.dot(df_brca_filtered_cov, diff))\n",
    "    mahalanobis_distance_abs = np.abs(distance) \n",
    "    distances.append(np.sqrt(mahalanobis_distance_abs))\n",
    "\n",
    "minimum_inter_luad_brca = np.min(distances)\n",
    "\n",
    "\n",
    "minimum_inter =  min(minimum_inter_luad_brca, minimum_inter_brca_luad)\n",
    "overlap_brca_luad = (max_intra_dist_brca + max_intra_dist_luad)/(2*minimum_inter)\n",
    "overlap_brca_luad"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}