09-Task-Evaluate_Data/01b_PRE_EEG_Metadata_Check.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7c9a9ba0-1fa4-467c-be6d-de2c92c50a30",
   "metadata": {},
   "source": [
    "# Check Metadata EEG - Impedances\n",
    "Status: Implemented\n",
    " -- Fully automatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "774c436e-1ab4-477b-ba08-dcf2008f1774",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "from utils.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f20b29fc-62f6-4f66-8a9a-edf6531bce21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all files from participants\n",
    "folders = get_participant_folder_per_participant()\n",
    "eeg_files = get_eeg_files_per_participant()\n",
    "assert (set(folders.keys()) == set(eeg_files.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "612341d1",
   "metadata": {},
   "source": [
    "### Check Impedance level of electrodes\n",
    "* for 4 EOG: less than 10 kOhm\n",
    "* for others: less than 5 kOhm\n",
    "\n",
    "max. 2 errors allowed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e0e71de7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bd4157faad148df9cc1cc0a446b118f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 003\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E003\\AoCfrp_003.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "2 errors with impedances, maximum breach of 3.0, sum of breaches in total 4.0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 004\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E004\\AoCfrp_004.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 005\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E005\\AoCfrp_005.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 007\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E007\\AoCfrp_007.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "1 errors with impedances, maximum breach of 2.0, sum of breaches in total 2.0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 008\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E008\\AoCfrp_008.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 009\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E009\\AoCfrp_009.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 010\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E010\\AoCfrp_010.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 011\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E011\\AoCfrp_011.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 012\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E012\\AoCfrp_012.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 013\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E013\\AoCfrp_013.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 014\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E014\\AoCfrp_014.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 015\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E015\\AoCfrp_015.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 016\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E016\\AoCfrp_016.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 017\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E017\\AoCfrp_017.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "1 errors with impedances, maximum breach of 2.0, sum of breaches in total 2.0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 018\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E018\\AoCfrp_018.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "2 errors with impedances, maximum breach of 3.0, sum of breaches in total 4.0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 019\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E019\\AoCfrp_019.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 020\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E020\\AoCfrp_020.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 021\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E021\\AoCfrp_021.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 022\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E022\\AoCfrp_022.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 023\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E023\\AoCfrp_0023.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 024\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E024\\AoCfrp_024.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 101\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E101\\AoCfrp_101.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 102\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E102\\AoCfrp_102.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n",
      "----------------------------------------------\n",
      "Participant 106\n",
      "(01/06) Load EEG data\n",
      "Extracting parameters from ..\\08-Data-Trial_Recordings\\raw\\E106\\AoCfrp_106.vhdr...\n",
      "Setting channel info structure...\n",
      "(02/06) Load impedances\n",
      "(03/06) Set boundaries for impedance values\n",
      "(04/06) Check impedances\n",
      "0 errors with impedances, maximum breach of 0, sum of breaches in total 0.\n",
      "\tImpedances accepted: True\n",
      "(06/06) Save impedances\n"
     ]
    }
   ],
   "source": [
    "# Iterate over all participants to check\n",
    "impedance_data_all = {}\n",
    "\n",
    "\n",
    "for participant in tqdm(folders):\n",
    "\n",
    "    print('----------------------------------------------')\n",
    "    print(PARTICIPANT, participant)\n",
    "    eeg_file = eeg_files[participant]\n",
    "\n",
    "    print('(01/06) Load EEG data')\n",
    "    raw_eeg = mne.io.read_raw_brainvision(\n",
    "        eeg_file[EEG_FILE_HEADER_ENDING], EOG_CHANNELS, preload=False)\n",
    "    sfreq = raw_eeg.info[MNE_KEY_FREQUENCY]\n",
    "\n",
    "    print('(02/06) Load impedances')\n",
    "    impedances = raw_eeg.impedances\n",
    "    if not impedances:\n",
    "        print(\n",
    "            f'\\tImpedances cannot be found and extracted automatically from file {eeg_file[EEG_FILE_HEADER_ENDING]}. Please check manually.')\n",
    "        continue\n",
    "    impedance_data = pd.DataFrame.from_dict(raw_eeg.impedances, orient='index')\n",
    "\n",
    "    print('(03/06) Set boundaries for impedance values')\n",
    "    # EOG less than 10, others less than 5\n",
    "    impedance_data[IMPEDANCE_UPPER_BOUND] = 5\n",
    "    for eog_c in EOG_CHANNELS:\n",
    "        impedance_data.at[eog_c, IMPEDANCE_UPPER_BOUND] = 10\n",
    "\n",
    "    print('(04/06) Check impedances')\n",
    "    # check which channel impedances exceed the upper boundary\n",
    "    impedance_okay = check_impedances(impedance_data, log=True)\n",
    "    if not impedance_okay:\n",
    "        print('(05/06) Update exclusions')\n",
    "        update_exclusions(participant, PARTICIPANT, EEG, {\n",
    "                          '': '01b_PRE_EEG_Metadata_Check: Impedance not acceptable.'})\n",
    "\n",
    "    print('(06/06) Save impedances')\n",
    "    impedance_data[PARTICIPANT] = participant\n",
    "    impedance_data_all[participant] = impedance_data\n",
    "    impedance_data.to_csv(get_impedance_path(participant), sep=SEPARATOR)\n",
    "\n",
    "impedance_data = pd.concat(impedance_data_all.values())\n",
    "impedance_data.to_csv(get_impedance_path(), sep=SEPARATOR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87c223d1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}