colocalization_signifcanceAndPlots.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import statsmodels.stats\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.metrics import pairwise_distances\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "savedStats='/media/xinyi/dcis2idc/plots/cnnvaeexp0/cluster_reordered/minibatchkmean_ncluster8n_pcs50epoch311_plottingIdx_progBalanced_0_reordered_plots/neighborHist_withRandom_nsamples40000'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neighborHist_withRandomCancer adjacent normal breast tissue_significanceSeparated.csv\n",
      "neighborHist_withRandomHyperplasia_significanceSeparated.csv\n",
      "neighborHist_withRandomAtypical hyperplasia_significanceSeparated.csv\n",
      "neighborHist_withRandomBreast tissue_significanceSeparated.csv\n",
      "neighborHist_withRandomDuctal carcinoma in situ and breast tissue_significanceSeparated.csv\n",
      "neighborHist_withRandomDuctal carcinoma in situ with early infiltratio_significanceSeparated.csv\n",
      "neighborHist_withRandomDuctal carcinoma in situ_significanceSeparated.csv\n",
      "neighborHist_withRandomInvasive ductal carcinoma (breast tissue)_significanceSeparated.csv\n",
      "neighborHist_withRandomInvasive ductal carcinoma and breast tissue_significanceSeparated.csv\n",
      "neighborHist_withRandomInvasive ductal carcinoma_significanceSeparated.csv\n",
      "neighborHist_withRandomMicropapillary type ductal carcinoma in situ wi_significanceSeparated.csv\n"
     ]
    }
   ],
   "source": [
    "ncluster=8\n",
    "pvalues=np.array([])\n",
    "fc=np.array([])\n",
    "sample=np.array([])\n",
    "for f in os.listdir(savedStats):\n",
    "    if not 'significanceSeparated' in f:\n",
    "        continue\n",
    "    print(f)\n",
    "    p_curr=np.genfromtxt(os.path.join(savedStats,f),delimiter=' ').flatten()\n",
    "    res=np.genfromtxt(os.path.join(savedStats,f[:-26]+'res.csv'),delimiter=' ').flatten()\n",
    "    randMean=np.genfromtxt(os.path.join(savedStats,f[:-26]+'randMean.csv'),delimiter=' ').flatten()\n",
    "    fname=f[23:-26]\n",
    "    sample=np.concatenate((sample,np.repeat(fname,p_curr.size)))\n",
    "    fc=np.concatenate((fc,res/randMean))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "logfc=np.log2(fc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.7083247986093806"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vmax_abs=np.max(np.abs(logfc))\n",
    "vmax_abs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in np.unique(sample):\n",
    "    fidx=sample==f\n",
    "    logfc_f=logfc[fidx].reshape(ncluster,ncluster)\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(10, 10))\n",
    "    im = ax.imshow(logfc_f,cmap='seismic',vmin=-vmax_abs,vmax=vmax_abs)\n",
    "    ax.set_xticks(np.arange(ncluster))\n",
    "    ax.set_xticklabels(np.arange(ncluster))\n",
    "    ax.set_yticks(np.arange(ncluster))\n",
    "    ax.set_yticklabels(np.arange(ncluster))\n",
    "    ax.tick_params(axis='both', which='major', labelsize=18)\n",
    "    fig.colorbar(im)\n",
    "    fig.tight_layout()\n",
    "    plt.savefig(os.path.join(savedStats,f+'_fc_noSig.jpg'),dpi=300)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### separate by pathology clf "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>sampleName</th>\n",
       "      <th>true</th>\n",
       "      <th>predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>br1003a_1_cytokeratin_555_aSMA_647_hoechst_A1</td>\n",
       "      <td>Hyperplasia</td>\n",
       "      <td>Hyperplasia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>br1003a_1_cytokeratin_555_aSMA_647_hoechst_A2</td>\n",
       "      <td>Hyperplasia</td>\n",
       "      <td>Hyperplasia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>br1003a_1_cytokeratin_555_aSMA_647_hoechst_A4</td>\n",
       "      <td>Hyperplasia</td>\n",
       "      <td>Hyperplasia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>br1003a_1_cytokeratin_555_aSMA_647_hoechst_A5</td>\n",
       "      <td>Hyperplasia</td>\n",
       "      <td>Hyperplasia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>br1003a_1_cytokeratin_555_aSMA_647_hoechst_A6</td>\n",
       "      <td>Hyperplasia</td>\n",
       "      <td>Hyperplasia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>226</td>\n",
       "      <td>br301_7_gh2ax_647_cytokeratin_555_hoechst_D5</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "      <td>Hyperplasia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>227</td>\n",
       "      <td>br301_7_gh2ax_647_cytokeratin_555_hoechst_E1</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>228</td>\n",
       "      <td>br301_7_gh2ax_647_cytokeratin_555_hoechst_E2</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>229</td>\n",
       "      <td>br301_7_gh2ax_647_cytokeratin_555_hoechst_E5</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>230</td>\n",
       "      <td>br301_7_gh2ax_647_cytokeratin_555_hoechst_E6</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "      <td>DCIS with early infiltration</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>231 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0                                     sampleName  \\\n",
       "0             0  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A1   \n",
       "1             1  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A2   \n",
       "2             2  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A4   \n",
       "3             3  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A5   \n",
       "4             4  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A6   \n",
       "..          ...                                            ...   \n",
       "226         226   br301_7_gh2ax_647_cytokeratin_555_hoechst_D5   \n",
       "227         227   br301_7_gh2ax_647_cytokeratin_555_hoechst_E1   \n",
       "228         228   br301_7_gh2ax_647_cytokeratin_555_hoechst_E2   \n",
       "229         229   br301_7_gh2ax_647_cytokeratin_555_hoechst_E5   \n",
       "230         230   br301_7_gh2ax_647_cytokeratin_555_hoechst_E6   \n",
       "\n",
       "                             true                     predicted  \n",
       "0                     Hyperplasia                   Hyperplasia  \n",
       "1                     Hyperplasia                   Hyperplasia  \n",
       "2                     Hyperplasia                   Hyperplasia  \n",
       "3                     Hyperplasia                   Hyperplasia  \n",
       "4                     Hyperplasia                   Hyperplasia  \n",
       "..                            ...                           ...  \n",
       "226  DCIS with early infiltration                   Hyperplasia  \n",
       "227  DCIS with early infiltration  DCIS with early infiltration  \n",
       "228  DCIS with early infiltration  DCIS with early infiltration  \n",
       "229  DCIS with early infiltration  DCIS with early infiltration  \n",
       "230  DCIS with early infiltration  DCIS with early infiltration  \n",
       "\n",
       "[231 rows x 4 columns]"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clfsavedir='/media/xinyi/dcis2idc/plots/cnnvaeexp0_pathologyClf_neighbor_clusters_exp0_subset_neighborOnly_crossVal_countAreaNorm_DCIS_AH_H//'\n",
    "predictions=pd.read_csv(clfsavedir+'/predictions.csv')\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir='/media/xinyi/dcis2idc/data'\n",
    "name='exp0'\n",
    "plotsavepath='/media/xinyi/dcis2idc/plots/cnnvae'+name\n",
    "sampledir=plotsavepath\n",
    "clustersavedir_alltrain=os.path.join(sampledir,'cluster_alltrain_reordered')\n",
    "\n",
    "with open(os.path.join(datadir,'processed','train_cnnvae_names'), 'rb') as input:\n",
    "    allImgNames=pickle.load(input)\n",
    "    \n",
    "with open(os.path.join(datadir,'processed','train_cnnvae_coord'), 'rb') as output:\n",
    "    coordlist=pickle.load(output)\n",
    "    \n",
    "with open(os.path.join(datadir,'processed','train_cnnvae_cellLabels'), 'rb') as output:\n",
    "    cellIDlist=pickle.load(output)\n",
    "    \n",
    "br1003aSpecs=pd.read_excel('/media/xinyi/dcis2idc/data/BR1003a specs.xlsx',header=10)\n",
    "br301Specs=pd.read_excel('/media/xinyi/dcis2idc/data/BR301 specs.xlsx',header=10)\n",
    "br8018aSpecs=pd.read_excel('/media/xinyi/dcis2idc/data/BR8018a specs.xlsx',header=10)\n",
    "br1003aSpecs.index=br1003aSpecs.loc[:,'Position']\n",
    "br301Specs.index=br301Specs.loc[:,'Position']\n",
    "br8018aSpecs.index=br8018aSpecs.loc[:,'Position']\n",
    "\n",
    "progList=np.copy(allImgNames)\n",
    "for s in np.unique(allImgNames):\n",
    "    ssplit=s.split('_')\n",
    "    if 'br1003a'==ssplit[0]:\n",
    "        prog_s=br1003aSpecs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    elif 'br301'==ssplit[0]:\n",
    "        prog_s=br301Specs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    elif 'br8018a'==ssplit[0]:\n",
    "        prog_s=br8018aSpecs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    progList[allImgNames==s]=prog_s\n",
    "    \n",
    "ncluster=8\n",
    "\n",
    "plotepoch=311\n",
    "clusterplotdir=os.path.join(clustersavedir_alltrain,'plots')\n",
    "n_pcs=50\n",
    "savenamecluster='minibatchkmean_ncluster'+str(ncluster)+'n_pcs'+str(n_pcs)+'epoch'+str(plotepoch)\n",
    "with open(os.path.join(clustersavedir_alltrain,'minibatchkmean_ncluster'+str(ncluster)+'n_pcs'+str(n_pcs)+'epoch'+str(plotepoch)+'_all'), 'rb') as output:\n",
    "    clusterRes=pickle.load(output)\n",
    "    \n",
    "for p in np.unique(progList):\n",
    "    if p=='Ductal carcinoma in situ':\n",
    "        progList[progList==p]='DCIS and breast tissue'\n",
    "    elif p=='Ductal carcinoma in situ and breast tissue':\n",
    "        progList[progList==p]='DCIS and breast tissue'\n",
    "    elif p=='Ductal carcinoma in situ with early infiltratio':\n",
    "        progList[progList==p]='DCIS with early infiltration'\n",
    "    \n",
    "    elif p=='Micropapillary type ductal carcinoma in situ wi':\n",
    "        progList[progList==p]='DCIS with early infiltration'    \n",
    "#     elif p=='Atypical hyperlasia':\n",
    "#         progList[progList==p]='Hyperplasia'\n",
    "        \n",
    "progInclude=np.array(['DCIS and breast tissue',  'DCIS with early infiltration', 'Hyperplasia','Atypical hyperplasia'])\n",
    "\n",
    "progIncludeIdx=np.repeat(False,progList.size)\n",
    "for p in progInclude:\n",
    "    progIncludeIdx[progList==p]=True\n",
    "    \n",
    "coordlist=coordlist[progIncludeIdx]\n",
    "allImgNames=allImgNames[progIncludeIdx]\n",
    "clusterRes=clusterRes[progIncludeIdx]\n",
    "progList=progList[progIncludeIdx]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "clustersavedir_valcores=os.path.join(sampledir,'cluster_valcores_reordered')\n",
    "clustersavedir_valsamples=os.path.join(sampledir,'cluster_valsamples_reordered')\n",
    "\n",
    "with open(os.path.join(datadir,'processed','train_cnnvae_coord_valcores'), 'rb') as output:\n",
    "    coordlist_valcores=pickle.load(output)\n",
    "with open(os.path.join(datadir,'processed','train_cnnvae_coord_valsamples'), 'rb') as output:\n",
    "    coordlist_valsamples=pickle.load(output)\n",
    "\n",
    "savenamecluster='minibatchkmean_ncluster'+str(ncluster)+'n_pcs'+str(n_pcs)+'epoch'+str(plotepoch)\n",
    "with open(os.path.join(clustersavedir_valcores,savenamecluster+'_all'), 'rb') as output:\n",
    "    clusterRes_valcores=pickle.load(output)\n",
    "with open(os.path.join(clustersavedir_valsamples,'minibatchkmean_ncluster'+str(ncluster)+'n_pcs'+str(n_pcs)+'epoch'+str(plotepoch)+'_all'), 'rb') as output:\n",
    "    clusterRes_valsamples=pickle.load(output)\n",
    "\n",
    "with open(os.path.join(datadir,'processed','train_cnnvae_names_valcores'), 'rb') as input:\n",
    "    allImgNames_valcores=pickle.load(input)\n",
    "with open(os.path.join(datadir,'processed','train_cnnvae_names_valsamples'), 'rb') as input:\n",
    "    allImgNames_valsamples=pickle.load(input)\n",
    "    \n",
    "br1003aSpecs=pd.read_excel('/media/xinyi/dcis2idc/data/BR1003a specs.xlsx',header=10)\n",
    "br301Specs=pd.read_excel('/media/xinyi/dcis2idc/data/BR301 specs.xlsx',header=10)\n",
    "br8018aSpecs=pd.read_excel('/media/xinyi/dcis2idc/data/BR8018a specs.xlsx',header=10)\n",
    "br1003aSpecs.index=br1003aSpecs.loc[:,'Position']\n",
    "br301Specs.index=br301Specs.loc[:,'Position']\n",
    "br8018aSpecs.index=br8018aSpecs.loc[:,'Position']\n",
    "\n",
    "progList_valcores=np.copy(allImgNames_valcores)\n",
    "for s in np.unique(allImgNames_valcores):\n",
    "    ssplit=s.split('_')\n",
    "    if 'br1003a'==ssplit[0]:\n",
    "        prog_s=br1003aSpecs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    elif 'br301'==ssplit[0]:\n",
    "        prog_s=br301Specs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    elif 'br8018a'==ssplit[0]:\n",
    "        prog_s=br8018aSpecs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    progList_valcores[allImgNames_valcores==s]=prog_s\n",
    "    \n",
    "progList_valsamples=np.copy(allImgNames_valsamples)\n",
    "for s in np.unique(allImgNames_valsamples):\n",
    "    ssplit=s.split('_')\n",
    "    if 'br1003a'==ssplit[0]:\n",
    "        prog_s=br1003aSpecs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    elif 'br301'==ssplit[0]:\n",
    "        prog_s=br301Specs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    elif 'br8018a'==ssplit[0]:\n",
    "        prog_s=br8018aSpecs.loc[(ssplit[-1],'Pathology diagnosis')]\n",
    "    progList_valsamples[allImgNames_valsamples==s]=prog_s\n",
    "    \n",
    "for p in np.unique(progList_valcores):\n",
    "    if p=='Ductal carcinoma in situ':\n",
    "        progList_valcores[progList_valcores==p]='DCIS and breast tissue'\n",
    "    elif p=='Ductal carcinoma in situ and breast tissue':\n",
    "        progList_valcores[progList_valcores==p]='DCIS and breast tissue'\n",
    "    elif p=='Ductal carcinoma in situ with early infiltratio':\n",
    "        progList_valcores[progList_valcores==p]='DCIS with early infiltration'\n",
    "    \n",
    "    elif p=='Micropapillary type ductal carcinoma in situ wi':\n",
    "        progList_valcores[progList_valcores==p]='DCIS with early infiltration'    \n",
    "#     elif p=='Atypical hyperlasia':\n",
    "#         progList_valcores[progList_valcores==p]='Hyperplasia'\n",
    "        \n",
    "for p in np.unique(progList_valsamples):\n",
    "    if p=='Ductal carcinoma in situ':\n",
    "        progList_valsamples[progList_valsamples==p]='DCIS and breast tissue'\n",
    "    elif p=='Ductal carcinoma in situ and breast tissue':\n",
    "        progList_valsamples[progList_valsamples==p]='DCIS and breast tissue'\n",
    "    elif p=='Ductal carcinoma in situ with early infiltrati':\n",
    "        progList_valsamples[progList_valsamples==p]='DCIS with early infiltration'\n",
    "    \n",
    "    elif p=='Micropapillary type ductal carcinoma in situ w':\n",
    "        progList_valsamples[progList_valsamples==p]='DCIS with early infiltration'    \n",
    "#     elif p=='Atypical hyperlasia':\n",
    "#         progList_valsamples[progList_valsamples==p]='Hyperplasia'\n",
    "        \n",
    "progIncludeIdx_valcores=np.repeat(False,progList_valcores.size)\n",
    "for p in progInclude:\n",
    "    progIncludeIdx_valcores[progList_valcores==p]=True\n",
    "    \n",
    "progIncludeIdx_valsamples=np.repeat(False,progList_valsamples.size)\n",
    "for p in progInclude:\n",
    "    progIncludeIdx_valsamples[progList_valsamples==p]=True\n",
    "    \n",
    "coordlist_valcores=coordlist_valcores[progIncludeIdx_valcores]\n",
    "allImgNames_valcores=allImgNames_valcores[progIncludeIdx_valcores]\n",
    "clusterRes_valcores=clusterRes_valcores[progIncludeIdx_valcores]\n",
    "progList_valcores=progList_valcores[progIncludeIdx_valcores]\n",
    "\n",
    "coordlist_valsamples=coordlist_valsamples[progIncludeIdx_valsamples]\n",
    "allImgNames_valsamples=allImgNames_valsamples[progIncludeIdx_valsamples]\n",
    "clusterRes_valsamples=clusterRes_valsamples[progIncludeIdx_valsamples]\n",
    "progList_valsamples=progList_valsamples[progIncludeIdx_valsamples]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "allImgNames_all=np.concatenate((allImgNames,np.concatenate((allImgNames_valcores,allImgNames_valsamples))))\n",
    "coordlist_all=np.concatenate((coordlist,np.concatenate((coordlist_valcores,coordlist_valsamples),axis=0)),axis=0)\n",
    "clusterRes_all=np.concatenate((clusterRes,np.concatenate((clusterRes_valcores,clusterRes_valsamples))))\n",
    "progList_all=np.concatenate((progList,np.concatenate((progList_valcores,progList_valsamples))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getHistMatrix(labels,ctlist):\n",
    "    res=np.zeros((np.unique(labels).size,np.unique(ctlist).size))\n",
    "    for li in range(res.shape[0]):\n",
    "        l=np.unique(labels)[li]\n",
    "        nl=np.sum(labels==l)\n",
    "        ctlist_l=ctlist[labels==l]\n",
    "        for ci in range(res.shape[1]):\n",
    "            c=np.unique(ctlist)[ci]\n",
    "            res[li,ci]=np.sum(ctlist_l==c)\n",
    "#             res[li,ci]=np.sum(ctlist_l==c)/nl\n",
    "        res[li]=res[li]/nl\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true: Atypical hyperplasia\n",
      "predicted: Atypical hyperplasia\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C1\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C3\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C4\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C5\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C6\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D3\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D4\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D7\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C1\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C3\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C4\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C6\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D3\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D7\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D8\n",
      "  br1003a_3_collagen1_647_hoechst_C4\n",
      "  br1003a_3_collagen1_647_hoechst_C5\n",
      "  br1003a_3_collagen1_647_hoechst_C6\n",
      "  br1003a_3_collagen1_647_hoechst_D3\n",
      "  br1003a_3_collagen1_647_hoechst_D4\n",
      "  br1003a_3_collagen1_647_hoechst_D7\n",
      "  br1003a_3_collagen1_647_hoechst_D8\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C1\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C4\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C6\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D3\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D4\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D7\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D8\n",
      "predicted: DCIS and breast tissue\n",
      "  br1003a_3_collagen1_647_hoechst_D10\n",
      "predicted: DCIS with early infiltration\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D9\n",
      "predicted: Hyperplasia\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D10\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D4\n",
      "  br1003a_3_collagen1_647_hoechst_C1\n",
      "  br1003a_3_collagen1_647_hoechst_C3\n",
      "  br1003a_3_collagen1_647_hoechst_D9\n",
      "true: DCIS and breast tissue\n",
      "predicted: Atypical hyperplasia\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A1\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_B3\n",
      "predicted: DCIS and breast tissue\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A3\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_B2\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_B3\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_B4\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A3\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A6\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_B2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_B4\n",
      "  br301_6_collagen1_647_hoechst_A3\n",
      "  br301_6_collagen1_647_hoechst_A5\n",
      "  br301_6_collagen1_647_hoechst_B2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A3\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_B3\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_B4\n",
      "predicted: DCIS with early infiltration\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A5\n",
      "  br301_6_collagen1_647_hoechst_B4\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_B2\n",
      "predicted: Hyperplasia\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A4\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A6\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A1\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A4\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A5\n",
      "  br301_6_collagen1_647_hoechst_A1\n",
      "  br301_6_collagen1_647_hoechst_A4\n",
      "  br301_6_collagen1_647_hoechst_A6\n",
      "  br301_6_collagen1_647_hoechst_B3\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A1\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A4\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A5\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A6\n",
      "true: DCIS with early infiltration\n",
      "predicted: Atypical hyperplasia\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C6\n",
      "predicted: DCIS and breast tissue\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C5\n",
      "  br301_6_collagen1_647_hoechst_C5\n",
      "  br301_6_collagen1_647_hoechst_D2\n",
      "predicted: DCIS with early infiltration\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C1\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C2\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C3\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C4\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C6\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_D1\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_D2\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_D5\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_E1\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_E5\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_E6\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C3\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C4\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C5\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_D1\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_D2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_D5\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_D6\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_E1\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_E2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_E5\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_E6\n",
      "  br301_6_collagen1_647_hoechst_A2\n",
      "  br301_6_collagen1_647_hoechst_C1\n",
      "  br301_6_collagen1_647_hoechst_C2\n",
      "  br301_6_collagen1_647_hoechst_C3\n",
      "  br301_6_collagen1_647_hoechst_C4\n",
      "  br301_6_collagen1_647_hoechst_C6\n",
      "  br301_6_collagen1_647_hoechst_D1\n",
      "  br301_6_collagen1_647_hoechst_D5\n",
      "  br301_6_collagen1_647_hoechst_E1\n",
      "  br301_6_collagen1_647_hoechst_E2\n",
      "  br301_6_collagen1_647_hoechst_E5\n",
      "  br301_6_collagen1_647_hoechst_E6\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C1\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C3\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C4\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C5\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C6\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_D1\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_D2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_E1\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_E2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_E5\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_E6\n",
      "predicted: Hyperplasia\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A2\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_D6\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_E2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C1\n",
      "  br301_6_collagen1_647_hoechst_D6\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_D5\n",
      "true: Hyperplasia\n",
      "predicted: Atypical hyperplasia\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A1\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B9\n",
      "predicted: DCIS and breast tissue\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B8\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C7\n",
      "  br1003a_3_collagen1_647_hoechst_D6\n",
      "predicted: DCIS with early infiltration\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A6\n",
      "predicted: Hyperplasia\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A1\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A2\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A4\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A5\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A6\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A7\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A8\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A9\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B1\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B10\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B2\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B3\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B7\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B8\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B9\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C10\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C2\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C7\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C8\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C9\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D1\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D2\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A1\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A10\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A3\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A7\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A8\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B1\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B3\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B6\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B7\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B9\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C10\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C2\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C8\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C9\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D1\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D2\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D6\n",
      "  br1003a_3_collagen1_647_hoechst_A1\n",
      "  br1003a_3_collagen1_647_hoechst_A2\n",
      "  br1003a_3_collagen1_647_hoechst_A3\n",
      "  br1003a_3_collagen1_647_hoechst_A5\n",
      "  br1003a_3_collagen1_647_hoechst_A6\n",
      "  br1003a_3_collagen1_647_hoechst_A7\n",
      "  br1003a_3_collagen1_647_hoechst_A8\n",
      "  br1003a_3_collagen1_647_hoechst_A9\n",
      "  br1003a_3_collagen1_647_hoechst_B1\n",
      "  br1003a_3_collagen1_647_hoechst_B10\n",
      "  br1003a_3_collagen1_647_hoechst_B2\n",
      "  br1003a_3_collagen1_647_hoechst_B3\n",
      "  br1003a_3_collagen1_647_hoechst_B5\n",
      "  br1003a_3_collagen1_647_hoechst_B7\n",
      "  br1003a_3_collagen1_647_hoechst_B8\n",
      "  br1003a_3_collagen1_647_hoechst_B9\n",
      "  br1003a_3_collagen1_647_hoechst_C10\n",
      "  br1003a_3_collagen1_647_hoechst_C2\n",
      "  br1003a_3_collagen1_647_hoechst_C7\n",
      "  br1003a_3_collagen1_647_hoechst_C8\n",
      "  br1003a_3_collagen1_647_hoechst_C9\n",
      "  br1003a_3_collagen1_647_hoechst_D1\n",
      "  br1003a_3_collagen1_647_hoechst_D2\n",
      "  br1003a_3_collagen1_647_hoechst_D5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A3\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A4\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A7\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A8\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A9\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B1\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B3\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B6\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B7\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B8\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C7\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C8\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C9\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D1\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D6\n"
     ]
    }
   ],
   "source": [
    "neighborhoodSize=16*9\n",
    "if not os.path.exists(os.path.join(clfsavedir,'neighborhood')):\n",
    "    os.mkdir(os.path.join(clfsavedir,'neighborhood'))\n",
    "for p_true in np.unique(predictions['true']):\n",
    "    print('true: '+p_true)\n",
    "    for p_pred in np.unique(np.unique(predictions['predicted'][predictions['true']==p_true])):\n",
    "        print('predicted: '+p_pred)\n",
    "        \n",
    "#         coord_p=coordlist[plottingIdx_i.astype(int)][progList[plottingIdx_i.astype(int)]==progs[p]]\n",
    "#         cluster_p=clusterRes[progList[plottingIdx_i.astype(int)]==progs[p]]\n",
    "#         imgNames_p=allImgNames[plottingIdx_i.astype(int)][progList[plottingIdx_i.astype(int)]==progs[p]]\n",
    "\n",
    "        selfp=np.array([])\n",
    "        neighborp=np.array([])\n",
    "        \n",
    "        for imgN in np.unique(predictions['sampleName'][np.logical_and(predictions['true']==p_true,predictions['predicted']==p_pred)]):\n",
    "            print('  '+imgN)\n",
    "            nsamples=np.sum(allImgNames_all==imgN)\n",
    "            cluster_ps=clusterRes_all[allImgNames_all==imgN]\n",
    "            neighbors=np.tile(cluster_ps,(nsamples,1))\n",
    "            selfs=np.tile(cluster_ps.reshape((-1,1)),(1,nsamples))\n",
    "\n",
    "            dist=pairwise_distances(coordlist_all[allImgNames_all==imgN],n_jobs=-1)\n",
    "            distIn=np.logical_and(dist<neighborhoodSize,dist>0)\n",
    "            selfp=np.concatenate((selfp,selfs[distIn]))\n",
    "            neighborp=np.concatenate((neighborp,neighbors[distIn]))\n",
    "        res=getHistMatrix(selfp,neighborp)\n",
    "        np.savetxt(os.path.join(clfsavedir,'neighborhood',p_true+'2'+p_pred+'.csv'),res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true: Atypical hyperplasia\n",
      "predicted: DCIS and breast tissue\n",
      "predicted: DCIS with early infiltration\n",
      "predicted: Hyperplasia\n",
      "true: DCIS and breast tissue\n",
      "predicted: Atypical hyperplasia\n",
      "predicted: DCIS with early infiltration\n",
      "predicted: Hyperplasia\n",
      "true: DCIS with early infiltration\n",
      "predicted: Atypical hyperplasia\n",
      "predicted: DCIS and breast tissue\n",
      "predicted: Hyperplasia\n",
      "true: Hyperplasia\n",
      "predicted: Atypical hyperplasia\n",
      "predicted: DCIS and breast tissue\n",
      "predicted: DCIS with early infiltration\n"
     ]
    }
   ],
   "source": [
    "logfc_all=np.array([])\n",
    "for p_true in np.unique(predictions['true']):\n",
    "    print('true: '+p_true)\n",
    "    trueNeighborhood=np.genfromtxt(os.path.join(clfsavedir,'neighborhood',p_true+'2'+p_true+'.csv'),delimiter=' ')\n",
    "    for p_pred in np.unique(np.unique(predictions['predicted'][predictions['true']==p_true])):\n",
    "        if p_pred==p_true:\n",
    "            continue\n",
    "        print('predicted: '+p_pred)\n",
    "        predNeighborhood=np.genfromtxt(os.path.join(clfsavedir,'neighborhood',p_true+'2'+p_pred+'.csv'),delimiter=' ')\n",
    "        logfc=np.log2(predNeighborhood/trueNeighborhood)\n",
    "        logfc_all=np.concatenate((logfc_all,logfc.flatten()))\n",
    "        logfc[logfc==np.inf]=2.38\n",
    "        fig, ax = plt.subplots(figsize=(10, 10))\n",
    "        im = ax.imshow(logfc,cmap='seismic',vmin=-2.38,vmax=2.38)\n",
    "        ax.set_xticks(np.arange(ncluster))\n",
    "        ax.set_xticklabels(np.arange(ncluster))\n",
    "        ax.set_yticks(np.arange(ncluster))\n",
    "        ax.set_yticklabels(np.arange(ncluster))\n",
    "        ax.tick_params(axis='both', which='major', labelsize=18)\n",
    "        fig.colorbar(im)\n",
    "        fig.tight_layout()\n",
    "        plt.savefig(os.path.join(clfsavedir,'neighborhood',p_true+'2'+p_pred+'_logfc.pdf'))\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.372497552388522"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(logfc_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "212"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with open(os.path.join(datadir,'processed','train_cnnvae_names'), 'rb') as input:\n",
    "#     allImgNames=pickle.load(input)\n",
    "# with open(os.path.join(datadir,'processed','train_cnnvae_names_valcores'), 'rb') as input:\n",
    "#     allImgNames_valcores=pickle.load(input)\n",
    "# with open(os.path.join(datadir,'processed','train_cnnvae_names_valsamples'), 'rb') as input:\n",
    "#     allImgNames_valsamples=pickle.load(input)\n",
    "    \n",
    "# allImgNames_all=np.concatenate((allImgNames,np.concatenate((allImgNames_valcores,allImgNames_valsamples))))\n",
    "\n",
    "\n",
    "# np.unique(allImgNames_all).size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true: Atypical hyperplasia\n",
      "predicted: Atypical hyperplasia\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C1\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C3\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C4\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C5\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C6\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D3\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D4\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D7\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C1\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C3\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C4\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C6\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D3\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D7\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D8\n",
      "  br1003a_3_collagen1_647_hoechst_C4\n",
      "  br1003a_3_collagen1_647_hoechst_C5\n",
      "  br1003a_3_collagen1_647_hoechst_C6\n",
      "  br1003a_3_collagen1_647_hoechst_D3\n",
      "  br1003a_3_collagen1_647_hoechst_D4\n",
      "  br1003a_3_collagen1_647_hoechst_D7\n",
      "  br1003a_3_collagen1_647_hoechst_D8\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C1\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C4\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C6\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D3\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D4\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D7\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D8\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: DCIS and breast tissue\n",
      "  br1003a_3_collagen1_647_hoechst_D10\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: DCIS with early infiltration\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D9\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: Hyperplasia\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D10\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D4\n",
      "  br1003a_3_collagen1_647_hoechst_C1\n",
      "  br1003a_3_collagen1_647_hoechst_C3\n",
      "  br1003a_3_collagen1_647_hoechst_D9\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "true: DCIS and breast tissue\n",
      "predicted: Atypical hyperplasia\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A1\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_B3\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: DCIS and breast tissue\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A3\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_B2\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_B3\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_B4\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A3\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A6\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_B2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_B4\n",
      "  br301_6_collagen1_647_hoechst_A3\n",
      "  br301_6_collagen1_647_hoechst_A5\n",
      "  br301_6_collagen1_647_hoechst_B2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A3\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_B3\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_B4\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: DCIS with early infiltration\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A5\n",
      "  br301_6_collagen1_647_hoechst_B4\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_B2\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: Hyperplasia\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A4\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A6\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A1\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A4\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A5\n",
      "  br301_6_collagen1_647_hoechst_A1\n",
      "  br301_6_collagen1_647_hoechst_A4\n",
      "  br301_6_collagen1_647_hoechst_A6\n",
      "  br301_6_collagen1_647_hoechst_B3\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A1\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A4\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A5\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A6\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "true: DCIS with early infiltration\n",
      "predicted: Atypical hyperplasia\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C6\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: DCIS and breast tissue\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C5\n",
      "  br301_6_collagen1_647_hoechst_C5\n",
      "  br301_6_collagen1_647_hoechst_D2\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: DCIS with early infiltration\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C1\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C2\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C3\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C4\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_C6\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_D1\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_D2\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_D5\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_E1\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_E5\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_E6\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C3\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C4\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C5\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_D1\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_D2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_D5\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_D6\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_E1\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_E2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_E5\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_E6\n",
      "  br301_6_collagen1_647_hoechst_A2\n",
      "  br301_6_collagen1_647_hoechst_C1\n",
      "  br301_6_collagen1_647_hoechst_C2\n",
      "  br301_6_collagen1_647_hoechst_C3\n",
      "  br301_6_collagen1_647_hoechst_C4\n",
      "  br301_6_collagen1_647_hoechst_C6\n",
      "  br301_6_collagen1_647_hoechst_D1\n",
      "  br301_6_collagen1_647_hoechst_D5\n",
      "  br301_6_collagen1_647_hoechst_E1\n",
      "  br301_6_collagen1_647_hoechst_E2\n",
      "  br301_6_collagen1_647_hoechst_E5\n",
      "  br301_6_collagen1_647_hoechst_E6\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C1\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C3\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C4\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C5\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_C6\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_D1\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_D2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_E1\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_E2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_E5\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_E6\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: Hyperplasia\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_A2\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_D6\n",
      "  br301_4_cytokeratin_555_aSMA_647_hoechst_E2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_A2\n",
      "  br301_5_cytokeratin_555_ki67_647_hoechst_C1\n",
      "  br301_6_collagen1_647_hoechst_D6\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_A2\n",
      "  br301_7_gh2ax_647_cytokeratin_555_hoechst_D5\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "true: Hyperplasia\n",
      "predicted: Atypical hyperplasia\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A1\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B9\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: DCIS and breast tissue\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B8\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C7\n",
      "  br1003a_3_collagen1_647_hoechst_D6\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: DCIS with early infiltration\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A6\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n",
      "predicted: Hyperplasia\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A1\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A2\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A4\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A5\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A6\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A7\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A8\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_A9\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B1\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B10\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B2\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B3\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B7\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B8\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_B9\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C10\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C2\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C7\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C8\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_C9\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D1\n",
      "  br1003a_1_cytokeratin_555_aSMA_647_hoechst_D2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A1\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A10\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A3\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A7\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_A8\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B1\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B2\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B3\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B6\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B7\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_B9\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C10\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C2\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C8\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_C9\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D1\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D2\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D5\n",
      "  br1003a_2_cytokeratin_555_ki67_647_hoechst_D6\n",
      "  br1003a_3_collagen1_647_hoechst_A1\n",
      "  br1003a_3_collagen1_647_hoechst_A2\n",
      "  br1003a_3_collagen1_647_hoechst_A3\n",
      "  br1003a_3_collagen1_647_hoechst_A5\n",
      "  br1003a_3_collagen1_647_hoechst_A6\n",
      "  br1003a_3_collagen1_647_hoechst_A7\n",
      "  br1003a_3_collagen1_647_hoechst_A8\n",
      "  br1003a_3_collagen1_647_hoechst_A9\n",
      "  br1003a_3_collagen1_647_hoechst_B1\n",
      "  br1003a_3_collagen1_647_hoechst_B10\n",
      "  br1003a_3_collagen1_647_hoechst_B2\n",
      "  br1003a_3_collagen1_647_hoechst_B3\n",
      "  br1003a_3_collagen1_647_hoechst_B5\n",
      "  br1003a_3_collagen1_647_hoechst_B7\n",
      "  br1003a_3_collagen1_647_hoechst_B8\n",
      "  br1003a_3_collagen1_647_hoechst_B9\n",
      "  br1003a_3_collagen1_647_hoechst_C10\n",
      "  br1003a_3_collagen1_647_hoechst_C2\n",
      "  br1003a_3_collagen1_647_hoechst_C7\n",
      "  br1003a_3_collagen1_647_hoechst_C8\n",
      "  br1003a_3_collagen1_647_hoechst_C9\n",
      "  br1003a_3_collagen1_647_hoechst_D1\n",
      "  br1003a_3_collagen1_647_hoechst_D2\n",
      "  br1003a_3_collagen1_647_hoechst_D5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A3\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A4\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A7\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A8\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_A9\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B1\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B3\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B6\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B7\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_B8\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C10\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C7\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C8\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_C9\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D1\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D2\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D5\n",
      "  br1003a_4_cytokeratin_555_gh2ax_647_hoechst_D6\n",
      "[0. 1. 2. 3. 4. 5. 6. 7.]\n"
     ]
    }
   ],
   "source": [
    "neighborhoodSize=16*9\n",
    "if not os.path.exists(os.path.join(clfsavedir,'clusterComp')):\n",
    "    os.mkdir(os.path.join(clfsavedir,'clusterComp'))\n",
    "for p_true in np.unique(predictions['true']):\n",
    "    print('true: '+p_true)\n",
    "    for p_pred in np.unique(np.unique(predictions['predicted'][predictions['true']==p_true])):\n",
    "        print('predicted: '+p_pred)\n",
    "        \n",
    "#         coord_p=coordlist[plottingIdx_i.astype(int)][progList[plottingIdx_i.astype(int)]==progs[p]]\n",
    "#         cluster_p=clusterRes[progList[plottingIdx_i.astype(int)]==progs[p]]\n",
    "#         imgNames_p=allImgNames[plottingIdx_i.astype(int)][progList[plottingIdx_i.astype(int)]==progs[p]]\n",
    "\n",
    "        selfp=np.array([])\n",
    "        \n",
    "        for imgN in np.unique(predictions['sampleName'][np.logical_and(predictions['true']==p_true,predictions['predicted']==p_pred)]):\n",
    "            print('  '+imgN)\n",
    "            nsamples=np.sum(allImgNames_all==imgN)\n",
    "            cluster_ps=clusterRes_all[allImgNames_all==imgN]\n",
    "            selfp=np.concatenate((selfp,cluster_ps))\n",
    "        uniquec,res=np.unique(selfp,return_counts=True)\n",
    "        print(uniquec)\n",
    "        np.savetxt(os.path.join(clfsavedir,'clusterComp',p_true+'2'+p_pred+'.csv'),res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true: Atypical hyperplasia\n",
      "predicted: DCIS and breast tissue\n",
      "predicted: DCIS with early infiltration\n",
      "predicted: Hyperplasia\n",
      "true: DCIS and breast tissue\n",
      "predicted: Atypical hyperplasia\n",
      "predicted: DCIS with early infiltration\n",
      "predicted: Hyperplasia\n",
      "true: DCIS with early infiltration\n",
      "predicted: Atypical hyperplasia\n",
      "predicted: DCIS and breast tissue\n",
      "predicted: Hyperplasia\n",
      "true: Hyperplasia\n",
      "predicted: Atypical hyperplasia\n",
      "predicted: DCIS and breast tissue\n",
      "predicted: DCIS with early infiltration\n"
     ]
    }
   ],
   "source": [
    "logfc_all=np.array([])\n",
    "for p_true in np.unique(predictions['true']):\n",
    "    print('true: '+p_true)\n",
    "    trueNeighborhood=np.genfromtxt(os.path.join(clfsavedir,'clusterComp',p_true+'2'+p_true+'.csv'),delimiter=' ')\n",
    "    trueNeighborhood=trueNeighborhood/np.sum(trueNeighborhood)\n",
    "    for p_pred in np.unique(np.unique(predictions['predicted'][predictions['true']==p_true])):\n",
    "        if p_pred==p_true:\n",
    "            continue\n",
    "        print('predicted: '+p_pred)\n",
    "        predNeighborhood=np.genfromtxt(os.path.join(clfsavedir,'clusterComp',p_true+'2'+p_pred+'.csv'),delimiter=' ')\n",
    "        predNeighborhood=predNeighborhood/np.sum(predNeighborhood)\n",
    "        logfc=np.log2(predNeighborhood/trueNeighborhood)\n",
    "        logfc_all=np.concatenate((logfc_all,logfc.flatten()))\n",
    "        fig, ax = plt.subplots(figsize=(10, 10))\n",
    "        im = ax.imshow(logfc.reshape(1,8),cmap='seismic',vmin=-1.47,vmax=1.47)\n",
    "        ax.set_xticks(np.arange(ncluster))\n",
    "        ax.set_xticklabels(np.arange(ncluster))\n",
    "#         ax.set_yticks(np.arange(ncluster))\n",
    "#         ax.set_yticklabels(np.arange(ncluster))\n",
    "        ax.tick_params(axis='both', which='major', labelsize=18)\n",
    "        fig.colorbar(im)\n",
    "        fig.tight_layout()\n",
    "        plt.savefig(os.path.join(clfsavedir,'clusterComp',p_true+'2'+p_pred+'_logfc.pdf'))\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.4640335526638943"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(logfc_all)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}