analysis_FDs_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "563a227c-5546-49f1-9cd9-f97c01dbc2e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.stats as stats\n",
    "\n",
    "import csv\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a72fa8f4-b9b4-422f-892e-1cd38c312ce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_info = 'THE_WHEEL_12_31_2019_CHESTFINALSUBJECTS.xls'\n",
    "\n",
    "pt_info = pd.read_excel(filename_info)\n",
    "pt_info['label'] = pt_info['Group_Arb2'].map({0: 'control',\n",
    "                                              1: 'pah',\n",
    "                                              5: 'epah'})\n",
    "pt_info['SID'] = pt_info['SID'].astype(str).str.zfill(3)\n",
    "\n",
    "label_dict = dict(zip(pt_info['SID'].to_list(), pt_info['label'].to_list()))\n",
    "# label_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2b91ec61-9735-4487-9fb2-6fecaf505d72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing search in:\n",
      "C:\\Users\\andre\\Documents\\github\\fd_analysis\\FDs_data\n",
      "\n",
      "Compiled 14468 files!\n"
     ]
    }
   ],
   "source": [
    "# Compile all spreadsheets in relevant folder\n",
    "\n",
    "old_folder = 'FDs_data'\n",
    "filters_include = ['_fd']\n",
    "filters_exclude = ['-checkpoint', 'Visual']\n",
    "extension = 'csv'\n",
    "\n",
    "search_dir = os.path.join(os.getcwd(), old_folder)\n",
    "\n",
    "print(f'Performing search in:\\n{search_dir}\\n')\n",
    "\n",
    "csvFile_list = []\n",
    "for root, dirs_list, files_list in os.walk(search_dir):\n",
    "    for file_name in files_list:\n",
    "        if file_name[-3:] == extension and \\\n",
    "        all([f in root or f in file_name for f in filters_include]) and \\\n",
    "        not any([f in root or f in file_name for f in filters_exclude]):\n",
    "            file_name_path = os.path.join(root, file_name)\n",
    "            csvFile_list.append(file_name_path)\n",
    "\n",
    "sid_list = pt_info['SID'].to_list()\n",
    "\n",
    "csvFile_list = [csvFile for csvFile in csvFile_list\n",
    "                if any(sid == csvFile.split('\\\\')[-1][:3] for sid in sid_list)]\n",
    "\n",
    "print(f'Compiled {len(csvFile_list)} files!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "91b744c9-2c6c-48b7-a500-fbf9e0aabab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "lobe_list = ['whole']\n",
    "artvein_list = ['', 'artery', 'vein']\n",
    "offset_list = ['', '_allOffsets']\n",
    "\n",
    "for lobe in lobe_list:\n",
    "    df_list = []\n",
    "    for sid in sid_list:\n",
    "        df_dict = {\n",
    "            'SID':             sid,\n",
    "            'lobe':            lobe,\n",
    "            'label':           label_dict[sid],\n",
    "        }\n",
    "        for artvein in artvein_list:\n",
    "            for offset in offset_list:\n",
    "                files_matching = csvFile_list\n",
    "                \n",
    "                # Check for sid and lobe matching in file namee\n",
    "                files_matching = [csvFile for csvFile in files_matching\n",
    "                                      if sid == csvFile.split('\\\\')[-1][:3] and \\\n",
    "                                         lobe in csvFile.split('\\\\')[-1]]\n",
    "                \n",
    "                # Check for artvein matching in file name\n",
    "                if artvein:\n",
    "                    files_matching = [csvFile for csvFile in files_matching\n",
    "                                      if artvein.capitalize() in csvFile.split('\\\\')[-1]]\n",
    "                else:\n",
    "                    files_matching = [csvFile for csvFile in files_matching\n",
    "                                      if 'Artery' not in csvFile.split('\\\\')[-1] and \\\n",
    "                                         'Vein' not in csvFile.split('\\\\')[-1]]\n",
    "                \n",
    "                # Check for offset matching in complete path name\n",
    "                if offset:\n",
    "                    files_matching = [csvFile for csvFile in files_matching\n",
    "                                      if 'offset' in csvFile.lower()]\n",
    "                else:\n",
    "                    files_matching = [csvFile for csvFile in files_matching\n",
    "                                      if 'offset' not in csvFile.lower()]\n",
    "\n",
    "                artvein_label = 'UNSEP' if not artvein else artvein.upper()\n",
    "                for f in files_matching:\n",
    "                    try:\n",
    "                        with open(f) as csv_file:\n",
    "                            csv_reader = csv.reader(csv_file)\n",
    "                            csv_data = list(csv_reader)\n",
    "\n",
    "                        folder = f.split('\\\\')[-2]\n",
    "\n",
    "                        sc_arr = np.asarray(csv_data[4:]).astype(int)\n",
    "                        sizes, counts = sc_arr[:, 0], sc_arr[:, 1]\n",
    "\n",
    "                        sizes_log =  np.log(sizes)\n",
    "                        counts_log = np.log(counts)\n",
    "                        coeffs =     np.array([csv_data[1][1], csv_data[2][1]]).astype(float)\n",
    "\n",
    "                        correlation_matrix = np.corrcoef(sizes_log, counts_log)\n",
    "                        r_squared =       correlation_matrix[0, 1]**2\n",
    "                        residuals =       counts_log - np.polyval(coeffs, sizes_log)\n",
    "                        pairwise_slopes = (counts_log[1:] - counts_log[:-1]) / (sizes_log[1:] - sizes_log[:-1])\n",
    "\n",
    "                        df_dict[f'{folder}_{artvein_label}__Fractal_Dimension'] = float(csv_data[0][1])\n",
    "\n",
    "                        if '20Size' in f and 'ArtVein' not in f:\n",
    "                            zero_crossings = np.where(np.diff(np.sign(np.array(residuals))))[0]\n",
    "                            if len(zero_crossings) > 2:\n",
    "                                df_dict[f'{folder}_{artvein_label}__FD@root1'] = np.nan\n",
    "                                df_dict[f'{folder}_{artvein_label}__FD@root2'] = np.nan\n",
    "                                continue\n",
    "                            zero_crossings = zero_crossings[::-1]\n",
    "\n",
    "                            # Fits\n",
    "                            counts_poly = []\n",
    "                            for i, c in enumerate(zero_crossings):\n",
    "                                counts_poly.append(np.polynomial.polynomial.Polynomial.fit(x=sizes_log[c-1:c+3], y=counts_log[c-1:c+3], deg=1, domain=[]))\n",
    "\n",
    "                            df_dict[f'{folder}_{artvein_label}__FD@root1'] = -counts_poly[0].coef[1]\n",
    "                            df_dict[f'{folder}_{artvein_label}__FD@root2'] = -counts_poly[1].coef[1]\n",
    "                    except:\n",
    "                        print(f'ERROR:\\n{f}\\n')\n",
    "\n",
    "        df_list.append(df_dict)\n",
    "    calculated_features = pd.DataFrame(df_list)\n",
    "    calculated_features.to_csv(f'analysis_FDs_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab474219-04eb-437c-8e42-d65ac49fd910",
   "metadata": {},
   "source": [
    "### Statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b302ce6d-08be-449c-9dde-ffdf1ccb27ec",
   "metadata": {},
   "source": [
    "### DNN vs FACTOR Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "267da15d-3e93-4b52-85fd-c939824936a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "group = ['control', 'pah']\n",
    "features = [\n",
    "    'FDs_UNSEP__Fractal_Dimension',\n",
    "    'FDs_DNN_SCALE_UNSEP__Fractal_Dimension',\n",
    "    'FDs_FACTOR_0_5_SCALE_UNSEP__Fractal_Dimension',\n",
    "    'FDs_FACTOR_1_0_SCALE_UNSEP__Fractal_Dimension',\n",
    "    'FDs_FACTOR_1_5_SCALE_UNSEP__Fractal_Dimension',\n",
    "]\n",
    "\n",
    "df_list = []\n",
    "for i, feature in enumerate(features):\n",
    "    X1 = calculated_features[calculated_features['label'].eq(group[0])][feature]\n",
    "    X2 = calculated_features[calculated_features['label'].eq(group[1])][feature]\n",
    "\n",
    "    _, pvalue = stats.mannwhitneyu(X1, X2)\n",
    "    \n",
    "    df_dict = {\n",
    "        'Feature':             feature,\n",
    "        f'{group[0]}_count':   len(X1),\n",
    "        f'{group[0]}_median':  np.median(X1),\n",
    "        f'{group[1]}_count':   len(X2),\n",
    "        f'{group[1]}_median':  np.median(X2),\n",
    "        'Wilcoxon p value':    pvalue,\n",
    "    }\n",
    "    \n",
    "    df_list.append(df_dict)\n",
    "\n",
    "df_stats = pd.DataFrame(df_list)\n",
    "df_stats.to_csv(f'analysis_DNN.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19a8f987-1605-4985-a13f-399a30f7da4d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}