09-Task-Evaluate_Data/01d4_PRE_VIS_Fixation_Correction-it3.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform Fixation Correction Iteration 3\n",
    "Status: Implemented\n",
    " -- Fully automatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "from utils.utils import *\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# get all files from participants\n",
    "folders = get_participant_folder_per_participant()\n",
    "sequence_orders = get_sequence_order_per_participant()\n",
    "assert (set(folders.keys()) == set(sequence_orders.keys()))\n",
    "snippets = {p: list(sequence_orders[p][SNIPPET].values)\n",
    "            for p in sequence_orders}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fixation correction iteration: 3\n",
      "Previous Fixation correction iteration: 2\n"
     ]
    }
   ],
   "source": [
    "global FIXATION_CORRECTION_ITERATION \n",
    "FIXATION_CORRECTION_ITERATION[0] = 3\n",
    "print('Fixation correction iteration:', current_fixation_correction_iteration())\n",
    "print('Previous Fixation correction iteration:', previous_fixation_correction_iteration())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chars</th>\n",
       "      <th>EssentialChars</th>\n",
       "      <th>LoC</th>\n",
       "      <th>EssentialLoC</th>\n",
       "      <th>Max length LoC</th>\n",
       "      <th>Words</th>\n",
       "      <th>EssentialWords</th>\n",
       "      <th>CodeLines</th>\n",
       "      <th>EssentialCodeLines</th>\n",
       "      <th>number</th>\n",
       "      <th>version</th>\n",
       "      <th>Snippet_base</th>\n",
       "      <th>variant</th>\n",
       "      <th>Condition</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Snippet</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-clean-v0</th>\n",
       "      <td>49</td>\n",
       "      <td>33</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 10;', 'int V2 = 3;', 'boolean R = (...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v0</td>\n",
       "      <td>--v0</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-clean-v1</th>\n",
       "      <td>48</td>\n",
       "      <td>32</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 4;', 'int V2 = 2;', 'boolean R = (V...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v1</td>\n",
       "      <td>--v1</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-clean-v2</th>\n",
       "      <td>48</td>\n",
       "      <td>32</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 1;', 'int V2 = 5;', 'boolean R = (V...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v2</td>\n",
       "      <td>--v2</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-obf-v0</th>\n",
       "      <td>54</td>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>31</td>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 10;', 'int V2 = 3;', 'boolean R = (...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v0</td>\n",
       "      <td>--v0</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-obf-v1</th>\n",
       "      <td>54</td>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>32</td>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 4;', 'int V2 = 2;', 'boolean R = ( ...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v1</td>\n",
       "      <td>--v1</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-clean-v1</th>\n",
       "      <td>37</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>['int V1 = 5;', 'boolean R = (V1 &gt; 2);', 'V1--;']</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v1</td>\n",
       "      <td>9--v1</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-clean-v2</th>\n",
       "      <td>38</td>\n",
       "      <td>27</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>['int V1 = 8;', 'boolean R = (V1 &lt;= 7);', 'V1-...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v2</td>\n",
       "      <td>9--v2</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-obf-v0</th>\n",
       "      <td>35</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>['int V1 = 2;', 'boolean R = (V1-- == 1);']</td>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v0</td>\n",
       "      <td>9--v0</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-obf-v1</th>\n",
       "      <td>34</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>['int V1 = 5;', 'boolean R = (V1-- &gt; 2);']</td>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v1</td>\n",
       "      <td>9--v1</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-obf-v2</th>\n",
       "      <td>35</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>['int V1 = 8;', 'boolean R = (V1-- &lt;= 7);']</td>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v2</td>\n",
       "      <td>9--v2</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>144 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Chars  EssentialChars  LoC  EssentialLoC  Max length LoC  Words   \n",
       "Snippet                                                                       \n",
       "-clean-v0      49              33    3             3              26     16  \\\n",
       "-clean-v1      48              32    3             3              26     16   \n",
       "-clean-v2      48              32    3             3              26     16   \n",
       "-obf-v0        54              36    3             3              31     18   \n",
       "-obf-v1        54              36    3             3              32     18   \n",
       "...           ...             ...  ...           ...             ...    ...   \n",
       "9-clean-v1     37              26    3             3              21     11   \n",
       "9-clean-v2     38              27    3             3              22     11   \n",
       "9-obf-v0       35              25    2             2              24     10   \n",
       "9-obf-v1       34              24    2             2              23     10   \n",
       "9-obf-v2       35              25    2             2              24     10   \n",
       "\n",
       "            EssentialWords                                          CodeLines   \n",
       "Snippet                                                                         \n",
       "-clean-v0               10  ['int V1 = 10;', 'int V2 = 3;', 'boolean R = (...  \\\n",
       "-clean-v1               10  ['int V1 = 4;', 'int V2 = 2;', 'boolean R = (V...   \n",
       "-clean-v2               10  ['int V1 = 1;', 'int V2 = 5;', 'boolean R = (V...   \n",
       "-obf-v0                 10  ['int V1 = 10;', 'int V2 = 3;', 'boolean R = (...   \n",
       "-obf-v1                 10  ['int V1 = 4;', 'int V2 = 2;', 'boolean R = ( ...   \n",
       "...                    ...                                                ...   \n",
       "9-clean-v1               7  ['int V1 = 5;', 'boolean R = (V1 > 2);', 'V1--;']   \n",
       "9-clean-v2               7  ['int V1 = 8;', 'boolean R = (V1 <= 7);', 'V1-...   \n",
       "9-obf-v0                 6        ['int V1 = 2;', 'boolean R = (V1-- == 1);']   \n",
       "9-obf-v1                 6         ['int V1 = 5;', 'boolean R = (V1-- > 2);']   \n",
       "9-obf-v2                 6        ['int V1 = 8;', 'boolean R = (V1-- <= 7);']   \n",
       "\n",
       "           EssentialCodeLines  number version Snippet_base variant  Condition  \n",
       "Snippet                                                                        \n",
       "-clean-v0           [1, 1, 1]       0      v0         --v0   clean      clean  \n",
       "-clean-v1           [1, 1, 1]       0      v1         --v1   clean      clean  \n",
       "-clean-v2           [1, 1, 1]       0      v2         --v2   clean      clean  \n",
       "-obf-v0             [1, 1, 1]       0      v0         --v0     obf  confusing  \n",
       "-obf-v1             [1, 1, 1]       0      v1         --v1     obf  confusing  \n",
       "...                       ...     ...     ...          ...     ...        ...  \n",
       "9-clean-v1          [1, 1, 1]       9      v1        9--v1   clean      clean  \n",
       "9-clean-v2          [1, 1, 1]       9      v2        9--v2   clean      clean  \n",
       "9-obf-v0               [1, 1]       9      v0        9--v0     obf  confusing  \n",
       "9-obf-v1               [1, 1]       9      v1        9--v1     obf  confusing  \n",
       "9-obf-v2               [1, 1]       9      v2        9--v2     obf  confusing  \n",
       "\n",
       "[144 rows x 14 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "code_sizes = pd.read_csv(CODE_SIZE_PATH, sep=SEPARATOR,\n",
    "                         index_col=False).set_index(SNIPPET)\n",
    "display(code_sizes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Take into account previous iteration of manual evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There exist manual accuracy evaluation files from the previous iteration which will be considered for this iteration's fixation correction. \n",
      "{'003': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/003_DataAccuracy_manual_evaluation_combined.csv'), '004': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/004_DataAccuracy_manual_evaluation_combined.csv'), '005': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/005_DataAccuracy_manual_evaluation_combined.csv'), '007': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/007_DataAccuracy_manual_evaluation_combined.csv'), '008': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/008_DataAccuracy_manual_evaluation_combined.csv'), '009': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/009_DataAccuracy_manual_evaluation_combined.csv'), '010': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/010_DataAccuracy_manual_evaluation_combined.csv'), '011': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/011_DataAccuracy_manual_evaluation_combined.csv'), '012': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/012_DataAccuracy_manual_evaluation_combined.csv'), '013': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/013_DataAccuracy_manual_evaluation_combined.csv'), '014': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/014_DataAccuracy_manual_evaluation_combined.csv'), '015': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/015_DataAccuracy_manual_evaluation_combined.csv'), '016': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/016_DataAccuracy_manual_evaluation_combined.csv'), '017': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/017_DataAccuracy_manual_evaluation_combined.csv'), '018': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/018_DataAccuracy_manual_evaluation_combined.csv'), '019': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/019_DataAccuracy_manual_evaluation_combined.csv'), '020': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/020_DataAccuracy_manual_evaluation_combined.csv'), '021': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/021_DataAccuracy_manual_evaluation_combined.csv'), '022': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/022_DataAccuracy_manual_evaluation_combined.csv'), '023': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/023_DataAccuracy_manual_evaluation_combined.csv'), '024': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/024_DataAccuracy_manual_evaluation_combined.csv'), '101': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/101_DataAccuracy_manual_evaluation_combined.csv'), '102': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/102_DataAccuracy_manual_evaluation_combined.csv'), '106': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_2/combined/106_DataAccuracy_manual_evaluation_combined.csv')}\n",
      "----------------------------------------------\n",
      "Participant 003\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 004\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 005\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 007\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 008\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 009\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 010\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 011\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 012\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 013\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 014\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 015\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 016\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 017\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 018\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 019\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 020\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 021\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 022\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 023\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 024\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 101\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 102\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 106\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n"
     ]
    }
   ],
   "source": [
    "# check whether there already exist manual evaluation files\n",
    "manual_evaluation_paths = exist_previous_manual_accuracy_evaluation_paths()\n",
    "if manual_evaluation_paths and current_fixation_correction_iteration() > 1:\n",
    "    assert (set(folders.keys()) == set(manual_evaluation_paths.keys())), f'''folder: {folders.keys()}, \\neval paths: {manual_evaluation_paths.keys()}'''\n",
    "\n",
    "    print('There exist manual accuracy evaluation files from the previous iteration which will be considered for this iteration\\'s fixation correction. ')\n",
    "    print(manual_evaluation_paths)\n",
    "    for participant in manual_evaluation_paths:\n",
    "        print('----------------------------------------------')\n",
    "        print(PARTICIPANT, participant)\n",
    "\n",
    "        print('(01/04) Load accuracy evaluation file')\n",
    "        manual_evaluation = pd.read_csv(\n",
    "            manual_evaluation_paths[participant], sep=SEPARATOR, index_col=False, dtype={PARTICIPANT: str})\n",
    "\n",
    "        print('(02/04) Select those in need of rework')\n",
    "        to_rework = get_to_rework_manual_accuracy_evaluation(\n",
    "            manual_evaluation, participant, snippets[participant])\n",
    "\n",
    "        print('(03/04) Check that outliers were removed correctly and offsets were adapted')\n",
    "        check_outlier_offset_actions(to_rework, participant)\n",
    "\n",
    "        print('(04/04) Save snippets to rework')\n",
    "        # save snippet names to rework in file\n",
    "        to_rework[SNIPPET].to_csv(get_snippet_rework_paths(\n",
    "            participant), sep=SEPARATOR, index=False)\n",
    "        snippets[participant] = to_rework[SNIPPET].to_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Auto-correction\n",
    "* perform auto-correction with various algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da19473a774c45b28c3fd097168c8d69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 003\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 004\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 005\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "209d353d0c5244b5a2687fed63cd0ddf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86a3845ce8f24988954e5148b1812b95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b3c395dc64144debdcc4919e911cdaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 007\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bf832a0247b4c25837f305844cd0513",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc2014bb8fb84dc390be82810fbe0639",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "100de4293bf6401e94f1ac8c90bde0f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 008\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c3bf461d3d5496989a7b7594ea171dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c95092a2e8b14b8fa41d72dce4d8ccfe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "604727594fe041b7b0372953bbdfad7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 009\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f719b14c19b44705a5fa42242fda3b00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ae7978c792d407c989654bd87c61a15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c773eced1754cb5b9457099b3f57fb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 010\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49bec8503e0b4715bf494f45b0c765b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40cf1c473aa2478b9be9b5b738b48b98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f436b6e53f8b4461aaaabc827021470f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 011\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 012\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3da733b2cf274118865579f2f570dfdf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "018da87f5f824655b28aaad5b0c9d0c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6e4eae0a1a8446fb0322f4618671036",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 013\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 014\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97f604018bab45bf921115e640eef406",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f56db43095c47e58db5734b08e7c56a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a9dde129ea3453596857894170d5e55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 015\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 016\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 017\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f6b8a6d1001409e899a9742ac79a65e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a19088221ca4af9996704cf587729e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9db46d0d7e0c4f43976ac1d2c9a3f81b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 018\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 019\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 020\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ebdeee0b9704b3399a8eb02c3a99cea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a78aee67f19b487c861a6a7fb5347ca9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b1c313d100f4df49f4dc417ef3742df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 021\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 022\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 023\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 024\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 101\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 102\n",
      "no snippets to (re-)work found.\n",
      "----------------------------------------------\n",
      "Participant 106\n",
      "no snippets to (re-)work found.\n"
     ]
    }
   ],
   "source": [
    "if current_fixation_correction_iteration() > 1:\n",
    "    # Iterate over all participants\n",
    "    for participant in tqdm(folders):\n",
    "        print('----------------------------------------------')\n",
    "        print(PARTICIPANT, participant)\n",
    "        if not snippets[participant]:\n",
    "            print('no snippets to (re-)work found.')\n",
    "            continue\n",
    "\n",
    "        print('(01/04) Delete existing fixation corrections')\n",
    "        for snippet in tqdm(snippets[participant]):\n",
    "            fixation_correction_paths = get_all_fixation_correction_paths(\n",
    "                participant, snippet)\n",
    "            for p in fixation_correction_paths:\n",
    "                p.unlink()\n",
    "\n",
    "        print('(02/04) Retrieve fixation data')\n",
    "        participant_snippet_fixations = {}\n",
    "        not_available = set()\n",
    "        for snippet in tqdm(snippets[participant]):\n",
    "            snippet_fixations_path = get_fixations_path(\n",
    "                participant, snippet, no_outliers_if_exists=True, iteration=previous_fixation_correction_iteration)\n",
    "            if snippet_fixations_path.exists():\n",
    "                participant_snippet_fixations[snippet] = pd.read_csv(\n",
    "                    snippet_fixations_path, index_col=False, sep=SEPARATOR, dtype={PARTICIPANT: str})\n",
    "                participant_snippet_fixations[snippet].to_csv(get_fixations_path(\n",
    "                    participant, snippet, no_outliers_if_exists=False), index=False, sep=SEPARATOR)\n",
    "            else:\n",
    "                not_available.add(snippet)\n",
    "        if not_available:\n",
    "            update_exclusions(participant, SNIPPET, VISUAL, {snippet: f'''01d4_PRE_VIS_Fixation_Correction-it{\n",
    "                              current_fixation_correction_iteration()}: Fixation data not available''' for snippet in not_available})\n",
    "        for snippet in not_available:\n",
    "            snippets[participant].remove(snippet)\n",
    "\n",
    "        print('(03/04) Perform auto-correction')\n",
    "        corrected_snippet_fixations = calculate_corrected_fixations(\n",
    "            participant_snippet_fixations, code_sizes)\n",
    "\n",
    "        print('(04/04) Store corrected fixation data')\n",
    "        # save fixation data\n",
    "        for snippet in tqdm(corrected_snippet_fixations):\n",
    "            corrected_fixation_datasets = corrected_snippet_fixations[snippet]\n",
    "            for key, fixation_data in corrected_fixation_datasets.items():\n",
    "                if key == FIXATION_CORRECTION_ALGORITHM_ORIGINAL:\n",
    "                    continue\n",
    "                fixation_data.to_csv(get_fixations_path(\n",
    "                    participant, snippet, modified_version=key), index=False, sep=SEPARATOR)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}