09-Task-Evaluate_Data/01f1_PRE_ALL_Summary_Participant_Exclusion.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7c9a9ba0-1fa4-467c-be6d-de2c92c50a30",
   "metadata": {},
   "source": [
    "# Summary & Participant Exclusion\n",
    "Status: Implemented\n",
    " -- Fully automatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cace8cd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "from utils.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eaf57434",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all files from participants\n",
    "folders = get_participant_folder_per_participant()\n",
    "# add gaze?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6dbc7f24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0      4-clean-v0\n",
      "1      4-clean-v1\n",
      "2      4-clean-v2\n",
      "3        4-obf-v0\n",
      "4        4-obf-v1\n",
      "          ...    \n",
      "139     -clean-v1\n",
      "140     -clean-v2\n",
      "141       -obf-v0\n",
      "142       -obf-v1\n",
      "143       -obf-v2\n",
      "Name: filename, Length: 144, dtype: object\n"
     ]
    }
   ],
   "source": [
    "snippets_data = pd.read_csv(SNIPPETS_PATH, sep=SEPARATOR).fillna(0)\n",
    "snippets_data[SNIPPET_COLUMN_SNIPPET] = snippets_data[SNIPPET_COLUMN_SNIPPET].str.replace(\n",
    "    '\\'', '')\n",
    "snippets_data[SNIPPET_COLUMN_SNIPPET] = snippets_data[SNIPPET_COLUMN_SNIPPET].apply(\n",
    "    lambda value: value.split('.')[0])\n",
    "print(snippets_data[SNIPPET_COLUMN_SNIPPET])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56717332",
   "metadata": {},
   "source": [
    "### Summarize all exclusion results\n",
    "* summarize, also per category / per participant\n",
    "* check whether a participant has less than x(10 probably) trials not excluded --> if too little, exclude completely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4d0303f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2373beaa6ab44775a99fbc7308cd8942",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 003\n",
      "----------------------------------------------\n",
      "Participant 004\n",
      "----------------------------------------------\n",
      "Participant 005\n",
      "----------------------------------------------\n",
      "Participant 007\n",
      "----------------------------------------------\n",
      "Participant 008\n",
      "----------------------------------------------\n",
      "Participant 009\n",
      "----------------------------------------------\n",
      "Participant 010\n",
      "----------------------------------------------\n",
      "Participant 011\n",
      "----------------------------------------------\n",
      "Participant 012\n",
      "----------------------------------------------\n",
      "Participant 013\n",
      "----------------------------------------------\n",
      "Participant 014\n",
      "----------------------------------------------\n",
      "Participant 015\n",
      "----------------------------------------------\n",
      "Participant 016\n",
      "----------------------------------------------\n",
      "Participant 017\n",
      "----------------------------------------------\n",
      "Participant 018\n",
      "----------------------------------------------\n",
      "Participant 019\n",
      "----------------------------------------------\n",
      "Participant 020\n",
      "----------------------------------------------\n",
      "Participant 021\n",
      "----------------------------------------------\n",
      "Participant 022\n",
      "----------------------------------------------\n",
      "Participant 023\n",
      "----------------------------------------------\n",
      "Participant 024\n",
      "----------------------------------------------\n",
      "Participant 101\n",
      "----------------------------------------------\n",
      "Participant 102\n",
      "----------------------------------------------\n",
      "Participant 106\n"
     ]
    }
   ],
   "source": [
    "# summary BEHAVIORAL\n",
    "#  summary EEG & VISUAL together based on BEHAVIORAL, but also separate\n",
    "# per participant and snippet\n",
    "behavioral_exclusion_summary = pd.DataFrame([], columns=[\n",
    "                                            participant for participant in folders], index=snippets_data[SNIPPET_COLUMN_SNIPPET])\n",
    "all_exclusion_summary = pd.DataFrame([], columns=[\n",
    "                                     participant for participant in folders], index=snippets_data[SNIPPET_COLUMN_SNIPPET])\n",
    "all_exclusion_summary_plus_violations = pd.DataFrame([], columns=[\n",
    "                                     participant for participant in folders], index=snippets_data[SNIPPET_COLUMN_SNIPPET])\n",
    "for participant in tqdm(folders):\n",
    "    print('----------------------------------------------')\n",
    "    print(PARTICIPANT, participant)\n",
    "    # participant marker\n",
    "    e_p = get_exclusions(participant, [PARTICIPANT])[PARTICIPANT]\n",
    "    if e_p[BEHAVIORAL]:\n",
    "        behavioral_exclusion_summary[participant] = False\n",
    "    if e_p[BEHAVIORAL] or e_p[EEG] or e_p[VISUAL]:\n",
    "        all_exclusion_summary[participant] = False\n",
    "        all_exclusion_summary_plus_violations[participant] = False\n",
    "\n",
    "    # snippet marker\n",
    "    e_s = get_exclusions(participant, [SNIPPET])[SNIPPET]\n",
    "    snippet_violations_path = get_eeg_violations_path(participant)\n",
    "    if snippet_violations_path.exists():\n",
    "        snippet_violations = pd.read_csv(\n",
    "            snippet_violations_path, sep=SEPARATOR, index_col=0, dtype={PARTICIPANT: str})\n",
    "        snippet_violations = snippet_violations.dropna(axis=0, how='all')\n",
    "    for snippet in e_s:\n",
    "        behavioral_exclusion_summary.loc[snippet, participant] = e_s[snippet][BEHAVIORAL] if e_s[snippet][BEHAVIORAL] else True\n",
    "        exclusion_reasons = e_s[snippet][BEHAVIORAL] + \\\n",
    "            e_s[snippet][EEG] + e_s[snippet][VISUAL]\n",
    "        if not exclusion_reasons:\n",
    "            value = True\n",
    "        else:\n",
    "            value = str(exclusion_reasons)\n",
    "        all_exclusion_summary.loc[snippet, participant] = value\n",
    "        if snippet_violations_path.exists() and (snippet in snippet_violations.index):\n",
    "            eeg_violations = snippet_violations.loc[snippet].dropna().to_dict()\n",
    "            exclusion_reasons += [eeg_violations]\n",
    "        if not exclusion_reasons:\n",
    "            value = True\n",
    "        else:\n",
    "            value = str(exclusion_reasons)\n",
    "        all_exclusion_summary_plus_violations.loc[snippet, participant] = value\n",
    "\n",
    "behavioral_exclusion_summary.to_csv(\n",
    "    get_statistic_summary_path('behavioral'), sep=SEPARATOR)\n",
    "all_exclusion_summary.to_csv(get_statistic_summary_path('all'), sep=SEPARATOR)\n",
    "all_exclusion_summary_plus_violations.to_csv(get_statistic_summary_path('including EEG violations'), sep=SEPARATOR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7577d6d9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}