run_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "00a8928b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "016fa97b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/root/.conda/envs/pytorch/lib/python3.11/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: '/root/.conda/envs/pytorch/lib/python3.11/site-packages/torchvision/image.so: undefined symbol: _ZN5torch3jit17parseSchemaOrNameERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?\n",
      "  warn(\n",
      "Before dropping duplicates: (190784, 5)\n",
      "After dropping duplicates: (133755, 5)\n",
      "\n",
      "Combined dataset shape: (133755, 5)\n",
      "\n",
      "\n",
      " --- DONE: Combined datasets saved to: data/combined_data/SubtaskA/SubtaskA_train_monolingual.jsonl ---\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!python combine_datasets.py \\\n",
    "    --file1=\"data/original_data/SubtaskA/SubtaskA_train_monolingual.jsonl\" \\\n",
    "    --file2=\"data/original_data/SubtaskB/SubtaskB_train.jsonl\" \\\n",
    "    --output=\"data/combined_data/SubtaskA/SubtaskA_train_monolingual.jsonl\" \\\n",
    "    --strategy=\"multi2binary\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24b5509e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python combine_datasets.py \\\n",
    "    --file1=\"data/original_data/SubtaskA/SubtaskA_train_monolingual.jsonl\" \\\n",
    "    --file2=\"data/original_data/SubtaskB/SubtaskB_train.jsonl\" \\\n",
    "    --output=\"data/combined_data/SubtaskB/SubtaskB_train.jsonl\" \\\n",
    "    --strategy=\"binary2multi\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2f32980e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/root/.conda/envs/pytorch/lib/python3.11/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: '/root/.conda/envs/pytorch/lib/python3.11/site-packages/torchvision/image.so: undefined symbol: _ZN5torch3jit17parseSchemaOrNameERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?\n",
      "  warn(\n",
      "Using device: cuda\n",
      "Loading train data...\n",
      "Train/dev split... (df_train.shape: (119757, 5))\n",
      "Loading test data...\n",
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Finding max batch size for BERT | Using fabric: False...\n",
      "\n",
      "Testing batch size 2...\n",
      "Testing batch size 4...\n",
      "Testing batch size 8...\n",
      "Testing batch size 16...\n",
      "Testing batch size 32...\n",
      "Testing batch size 64...\n",
      "Batch size 64 failed with error:\n",
      "\n",
      "CUDA out of memory. Tried to allocate 768.00 MiB. GPU 0 has a total capacty of 10.75 GiB of which 329.62 MiB is free. Process 166917 has 10.43 GiB memory in use. Of the allocated memory 10.11 GiB is allocated by PyTorch, and 129.63 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n",
      "Max allowed batch size: 32\n"
     ]
    }
   ],
   "source": [
    "!python get_max_batch_size.py --fine-tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b1cf7069",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/root/.conda/envs/pytorch/lib/python3.11/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: '/root/.conda/envs/pytorch/lib/python3.11/site-packages/torchvision/image.so: undefined symbol: _ZN5torch3jit17parseSchemaOrNameERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?\n",
      "  warn(\n",
      "Using device: cuda\n",
      "Will save results to: runs/14-12-2023_01:45:56-SubtaskA-monolingual-bert\n",
      "Loading train data...\n",
      "Train/dev split... (df_train.shape: (119757, 5))\n",
      "Loading test data...\n",
      "df_train.shape: (95805, 5)\n",
      "df_dev.shape: (23952, 5)\n",
      "df_test.shape: (5000, 5)\n",
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Epoch 1/10\n",
      "Freeze transformeer\n",
      "--------------------\n",
      "Batch=[1/5988]; Loss=[0.71462]; Acc. Metric=0.5; Elapsed time=0.77s [00:00:00.77]\n",
      "Batch=[101/5988]; Loss=[0.48053]; Acc. Metric=0.875; Elapsed time=26.50s [00:00:26.50]\n",
      "Batch=[201/5988]; Loss=[0.65652]; Acc. Metric=0.625; Elapsed time=26.76s [00:00:26.76]\n",
      "Batch=[301/5988]; Loss=[0.50341]; Acc. Metric=0.75; Elapsed time=26.29s [00:00:26.29]\n",
      "Batch=[901/5988]; Loss=[0.24712]; Acc. Metric=1.0; Elapsed time=27.48s [00:00:27.48]\n",
      "Batch=[1001/5988]; Loss=[0.51298]; Acc. Metric=0.75; Elapsed time=26.96s [00:00:26.96]\n",
      "Batch=[1101/5988]; Loss=[0.45199]; Acc. Metric=0.75; Elapsed time=27.10s [00:00:27.10]\n",
      "Batch=[1201/5988]; Loss=[0.31957]; Acc. Metric=0.8125; Elapsed time=26.99s [00:00:26.99]\n",
      "Batch=[1301/5988]; Loss=[0.46849]; Acc. Metric=0.6875; Elapsed time=26.95s [00:00:26.95]\n",
      "Batch=[1401/5988]; Loss=[0.50096]; Acc. Metric=0.75; Elapsed time=27.88s [00:00:27.88]\n",
      "Batch=[1501/5988]; Loss=[0.20969]; Acc. Metric=1.0; Elapsed time=26.35s [00:00:26.35]\n",
      "Batch=[1601/5988]; Loss=[0.57664]; Acc. Metric=0.75; Elapsed time=27.35s [00:00:27.35]\n",
      "Batch=[1701/5988]; Loss=[0.38460]; Acc. Metric=0.8125; Elapsed time=27.01s [00:00:27.01]\n",
      "Batch=[1801/5988]; Loss=[0.57320]; Acc. Metric=0.8125; Elapsed time=26.98s [00:00:26.98]\n",
      "Batch=[1901/5988]; Loss=[0.92622]; Acc. Metric=0.5625; Elapsed time=26.87s [00:00:26.87]\n",
      "Batch=[2001/5988]; Loss=[0.54150]; Acc. Metric=0.6875; Elapsed time=27.09s [00:00:27.09]\n",
      "Batch=[2101/5988]; Loss=[0.43135]; Acc. Metric=0.75; Elapsed time=27.77s [00:00:27.77]\n",
      "Batch=[2201/5988]; Loss=[0.73876]; Acc. Metric=0.75; Elapsed time=27.54s [00:00:27.54]\n",
      "Batch=[2301/5988]; Loss=[0.49298]; Acc. Metric=0.6875; Elapsed time=26.84s [00:00:26.84]\n",
      "Batch=[2401/5988]; Loss=[0.34102]; Acc. Metric=0.8125; Elapsed time=27.06s [00:00:27.06]\n",
      "Batch=[2501/5988]; Loss=[0.35189]; Acc. Metric=0.875; Elapsed time=26.96s [00:00:26.96]\n",
      "Batch=[2601/5988]; Loss=[0.25105]; Acc. Metric=0.9375; Elapsed time=26.63s [00:00:26.63]\n",
      "Batch=[2701/5988]; Loss=[0.29230]; Acc. Metric=0.8125; Elapsed time=26.97s [00:00:26.97]\n",
      "Batch=[2801/5988]; Loss=[0.27647]; Acc. Metric=0.875; Elapsed time=27.25s [00:00:27.25]\n",
      "Batch=[2901/5988]; Loss=[0.62336]; Acc. Metric=0.625; Elapsed time=27.72s [00:00:27.72]\n",
      "Batch=[3001/5988]; Loss=[0.44444]; Acc. Metric=0.6875; Elapsed time=26.80s [00:00:26.80]\n",
      "Batch=[3101/5988]; Loss=[0.47483]; Acc. Metric=0.875; Elapsed time=27.89s [00:00:27.89]\n",
      "Batch=[3201/5988]; Loss=[0.58946]; Acc. Metric=0.75; Elapsed time=26.94s [00:00:26.94]\n",
      "Batch=[3301/5988]; Loss=[0.47364]; Acc. Metric=0.8125; Elapsed time=27.22s [00:00:27.22]\n",
      "Batch=[3401/5988]; Loss=[0.30126]; Acc. Metric=0.8125; Elapsed time=27.26s [00:00:27.26]\n",
      "Batch=[3501/5988]; Loss=[0.59346]; Acc. Metric=0.6875; Elapsed time=26.97s [00:00:26.97]\n",
      "Batch=[3601/5988]; Loss=[0.27332]; Acc. Metric=0.9375; Elapsed time=26.63s [00:00:26.63]\n",
      "Batch=[3701/5988]; Loss=[0.69691]; Acc. Metric=0.625; Elapsed time=26.90s [00:00:26.90]\n",
      "Batch=[3801/5988]; Loss=[0.30204]; Acc. Metric=0.8125; Elapsed time=26.95s [00:00:26.95]\n",
      "Batch=[3901/5988]; Loss=[0.56272]; Acc. Metric=0.6875; Elapsed time=26.55s [00:00:26.55]\n",
      "Batch=[4001/5988]; Loss=[0.68074]; Acc. Metric=0.625; Elapsed time=27.29s [00:00:27.29]\n",
      "Batch=[4101/5988]; Loss=[0.31769]; Acc. Metric=0.875; Elapsed time=26.71s [00:00:26.71]\n",
      "Batch=[4201/5988]; Loss=[0.23080]; Acc. Metric=0.9375; Elapsed time=26.93s [00:00:26.93]\n",
      "Batch=[4301/5988]; Loss=[0.29026]; Acc. Metric=0.875; Elapsed time=27.39s [00:00:27.39]\n",
      "Batch=[4401/5988]; Loss=[0.31627]; Acc. Metric=0.8125; Elapsed time=27.30s [00:00:27.30]\n",
      "Batch=[4501/5988]; Loss=[0.15636]; Acc. Metric=0.9375; Elapsed time=28.02s [00:00:28.02]\n",
      "Batch=[4601/5988]; Loss=[0.27010]; Acc. Metric=0.875; Elapsed time=26.74s [00:00:26.74]\n",
      "Batch=[4701/5988]; Loss=[0.37612]; Acc. Metric=0.8125; Elapsed time=26.63s [00:00:26.63]\n",
      "Batch=[4801/5988]; Loss=[0.58272]; Acc. Metric=0.75; Elapsed time=26.77s [00:00:26.77]\n",
      "Batch=[4901/5988]; Loss=[0.51849]; Acc. Metric=0.75; Elapsed time=27.20s [00:00:27.20]\n",
      "Batch=[5001/5988]; Loss=[0.57517]; Acc. Metric=0.75; Elapsed time=27.51s [00:00:27.51]\n",
      "Batch=[5101/5988]; Loss=[0.36622]; Acc. Metric=0.8125; Elapsed time=27.35s [00:00:27.35]\n",
      "Batch=[5201/5988]; Loss=[0.20402]; Acc. Metric=0.875; Elapsed time=27.46s [00:00:27.46]\n",
      "Batch=[5301/5988]; Loss=[0.53655]; Acc. Metric=0.6875; Elapsed time=27.03s [00:00:27.03]\n",
      "Batch=[5401/5988]; Loss=[0.54782]; Acc. Metric=0.6875; Elapsed time=27.50s [00:00:27.50]\n",
      "Batch=[5501/5988]; Loss=[0.23734]; Acc. Metric=0.9375; Elapsed time=27.60s [00:00:27.60]\n",
      "Batch=[5601/5988]; Loss=[0.40229]; Acc. Metric=0.75; Elapsed time=26.97s [00:00:26.97]\n",
      "Batch=[5701/5988]; Loss=[0.42561]; Acc. Metric=0.875; Elapsed time=27.45s [00:00:27.45]\n",
      "Batch=[5801/5988]; Loss=[0.62375]; Acc. Metric=0.6875; Elapsed time=27.95s [00:00:27.95]\n",
      "Batch=[5901/5988]; Loss=[0.58537]; Acc. Metric=0.6875; Elapsed time=27.12s [00:00:27.12]\n",
      "Train Loss: 0.42756; Train Metric: 0.79819; Elapsed time=1624.40s [00:27:04.40]\n",
      "100%|███████████████████████████████████████| 1497/1497 [06:23<00:00,  3.90it/s]\n",
      "Validation Loss: 0.36119; Validation Metric: 0.82782\n",
      "Epoch 2/10\n",
      "Freeze transformeer\n",
      "--------------------\n",
      "Batch=[1/5988]; Loss=[0.52158]; Acc. Metric=0.6875; Elapsed time=0.30s [00:00:00.30]\n",
      "Batch=[101/5988]; Loss=[0.32461]; Acc. Metric=0.875; Elapsed time=28.07s [00:00:28.07]\n",
      "Batch=[201/5988]; Loss=[0.55590]; Acc. Metric=0.75; Elapsed time=27.32s [00:00:27.32]\n",
      "Batch=[301/5988]; Loss=[0.29799]; Acc. Metric=0.875; Elapsed time=27.34s [00:00:27.34]\n",
      "Batch=[401/5988]; Loss=[0.30245]; Acc. Metric=0.8125; Elapsed time=26.86s [00:00:26.86]\n",
      "Batch=[501/5988]; Loss=[0.31577]; Acc. Metric=0.875; Elapsed time=27.24s [00:00:27.24]\n",
      "Batch=[601/5988]; Loss=[0.52016]; Acc. Metric=0.6875; Elapsed time=27.84s [00:00:27.84]\n",
      "Batch=[701/5988]; Loss=[0.35319]; Acc. Metric=0.75; Elapsed time=27.52s [00:00:27.52]\n",
      "Batch=[801/5988]; Loss=[0.19872]; Acc. Metric=0.875; Elapsed time=27.09s [00:00:27.09]\n",
      "Batch=[901/5988]; Loss=[0.63092]; Acc. Metric=0.8125; Elapsed time=26.91s [00:00:26.91]\n",
      "Batch=[1001/5988]; Loss=[0.35194]; Acc. Metric=0.8125; Elapsed time=26.81s [00:00:26.81]\n",
      "Batch=[1101/5988]; Loss=[0.56916]; Acc. Metric=0.75; Elapsed time=27.11s [00:00:27.11]\n",
      "Batch=[1201/5988]; Loss=[0.59842]; Acc. Metric=0.75; Elapsed time=27.44s [00:00:27.44]\n",
      "Batch=[1301/5988]; Loss=[0.36030]; Acc. Metric=0.8125; Elapsed time=27.48s [00:00:27.48]\n",
      "Batch=[1401/5988]; Loss=[0.43891]; Acc. Metric=0.75; Elapsed time=27.56s [00:00:27.56]\n",
      "Batch=[1501/5988]; Loss=[0.35322]; Acc. Metric=0.8125; Elapsed time=27.38s [00:00:27.38]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch=[1601/5988]; Loss=[0.30823]; Acc. Metric=0.875; Elapsed time=26.72s [00:00:26.72]\n",
      "Batch=[1701/5988]; Loss=[0.30742]; Acc. Metric=0.875; Elapsed time=27.34s [00:00:27.34]\n",
      "Batch=[1801/5988]; Loss=[0.25864]; Acc. Metric=0.875; Elapsed time=27.25s [00:00:27.25]\n",
      "Batch=[1901/5988]; Loss=[0.42680]; Acc. Metric=0.75; Elapsed time=26.99s [00:00:26.99]\n",
      "Batch=[2001/5988]; Loss=[0.42204]; Acc. Metric=0.6875; Elapsed time=26.76s [00:00:26.76]\n",
      "Batch=[2101/5988]; Loss=[0.60912]; Acc. Metric=0.75; Elapsed time=27.28s [00:00:27.28]\n",
      "Batch=[2201/5988]; Loss=[0.41335]; Acc. Metric=0.75; Elapsed time=26.96s [00:00:26.96]\n",
      "Batch=[2301/5988]; Loss=[0.41009]; Acc. Metric=0.75; Elapsed time=27.28s [00:00:27.28]\n",
      "Batch=[2401/5988]; Loss=[0.24671]; Acc. Metric=0.875; Elapsed time=27.09s [00:00:27.09]\n",
      "Batch=[2501/5988]; Loss=[0.19942]; Acc. Metric=0.9375; Elapsed time=27.46s [00:00:27.46]\n",
      "Batch=[2601/5988]; Loss=[0.48132]; Acc. Metric=0.8125; Elapsed time=27.02s [00:00:27.02]\n",
      "Batch=[2701/5988]; Loss=[0.24783]; Acc. Metric=0.9375; Elapsed time=27.65s [00:00:27.65]\n",
      "Batch=[2801/5988]; Loss=[0.15630]; Acc. Metric=0.9375; Elapsed time=26.90s [00:00:26.90]\n",
      "Batch=[2901/5988]; Loss=[0.55253]; Acc. Metric=0.6875; Elapsed time=26.89s [00:00:26.89]\n",
      "Batch=[3001/5988]; Loss=[0.25172]; Acc. Metric=0.8125; Elapsed time=26.75s [00:00:26.75]\n",
      "Batch=[3101/5988]; Loss=[0.40893]; Acc. Metric=0.8125; Elapsed time=26.95s [00:00:26.95]\n",
      "Batch=[3201/5988]; Loss=[0.20045]; Acc. Metric=0.875; Elapsed time=26.95s [00:00:26.95]\n",
      "Batch=[3301/5988]; Loss=[0.48486]; Acc. Metric=0.75; Elapsed time=27.04s [00:00:27.04]\n",
      "Batch=[3401/5988]; Loss=[0.69312]; Acc. Metric=0.6875; Elapsed time=26.87s [00:00:26.87]\n",
      "Batch=[3501/5988]; Loss=[0.47292]; Acc. Metric=0.8125; Elapsed time=27.48s [00:00:27.48]\n",
      "Batch=[3601/5988]; Loss=[0.24894]; Acc. Metric=0.875; Elapsed time=26.95s [00:00:26.95]\n",
      "Batch=[3701/5988]; Loss=[0.43596]; Acc. Metric=0.8125; Elapsed time=27.59s [00:00:27.59]\n",
      "Batch=[3801/5988]; Loss=[0.47862]; Acc. Metric=0.8125; Elapsed time=27.92s [00:00:27.92]\n",
      "Batch=[3901/5988]; Loss=[0.19439]; Acc. Metric=0.9375; Elapsed time=26.98s [00:00:26.98]\n",
      "Batch=[4001/5988]; Loss=[0.38574]; Acc. Metric=0.875; Elapsed time=27.36s [00:00:27.36]\n",
      "Batch=[4101/5988]; Loss=[0.26611]; Acc. Metric=0.8125; Elapsed time=26.55s [00:00:26.55]\n",
      "Batch=[4201/5988]; Loss=[0.18864]; Acc. Metric=0.9375; Elapsed time=27.10s [00:00:27.10]\n",
      "Batch=[4301/5988]; Loss=[0.26316]; Acc. Metric=0.9375; Elapsed time=27.23s [00:00:27.23]\n",
      "Batch=[4401/5988]; Loss=[0.35517]; Acc. Metric=0.8125; Elapsed time=27.31s [00:00:27.31]\n",
      "Batch=[4501/5988]; Loss=[0.19391]; Acc. Metric=0.9375; Elapsed time=27.05s [00:00:27.05]\n",
      "Batch=[4601/5988]; Loss=[0.49566]; Acc. Metric=0.8125; Elapsed time=27.33s [00:00:27.33]\n",
      "Batch=[4701/5988]; Loss=[0.48340]; Acc. Metric=0.8125; Elapsed time=27.17s [00:00:27.17]\n",
      "Batch=[4801/5988]; Loss=[0.47373]; Acc. Metric=0.8125; Elapsed time=27.59s [00:00:27.59]\n",
      "Batch=[4901/5988]; Loss=[0.43036]; Acc. Metric=0.8125; Elapsed time=27.79s [00:00:27.79]\n",
      "Batch=[5001/5988]; Loss=[0.32056]; Acc. Metric=0.875; Elapsed time=27.27s [00:00:27.27]\n",
      "Batch=[5101/5988]; Loss=[0.16449]; Acc. Metric=1.0; Elapsed time=26.66s [00:00:26.66]\n",
      "Batch=[5201/5988]; Loss=[0.33453]; Acc. Metric=0.875; Elapsed time=27.73s [00:00:27.73]\n",
      "Batch=[5301/5988]; Loss=[0.30745]; Acc. Metric=0.8125; Elapsed time=26.91s [00:00:26.91]\n",
      "Batch=[5401/5988]; Loss=[0.43940]; Acc. Metric=0.8125; Elapsed time=27.09s [00:00:27.09]\n",
      "Batch=[5501/5988]; Loss=[0.23647]; Acc. Metric=0.875; Elapsed time=26.67s [00:00:26.67]\n",
      "Batch=[5601/5988]; Loss=[0.17226]; Acc. Metric=0.9375; Elapsed time=26.71s [00:00:26.71]\n",
      "Batch=[5701/5988]; Loss=[0.40738]; Acc. Metric=0.8125; Elapsed time=27.41s [00:00:27.41]\n",
      "Batch=[5801/5988]; Loss=[0.61781]; Acc. Metric=0.8125; Elapsed time=27.67s [00:00:27.67]\n",
      "Batch=[5901/5988]; Loss=[0.27314]; Acc. Metric=0.875; Elapsed time=26.78s [00:00:26.78]\n",
      "Train Loss: 0.38792; Train Metric: 0.82320; Elapsed time=1628.37s [00:27:08.37]\n",
      " 93%|████████████████████████████████████▏  | 1389/1497 [05:55<00:29,  3.70it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!python main.py --save-results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ba012020",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results on validation\n",
      "Accuracy: 96.03%\n",
      "Precision: 92.61%\n",
      "Recall: 99.49%\n",
      "F1: 95.93%\n",
      "--------------------\n",
      "Results on test\n",
      "Accuracy: 80.98%\n",
      "Precision: 89.29%\n",
      "Recall: 70.40%\n",
      "F1: 78.73%\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "!python scores_and_plots.py --results-dir \"runs/22-11-2023_17:32:00-SubtaskA-monolingual-bert\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c693297e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python scores_and_plots.py --results-dir \"runs/23-11-2023_07:07:12-SubtaskA-multilingual-bert\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6b23dcd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results on validation\n",
      "Accuracy: 99.71%\n",
      "Precision: 99.64%\n",
      "Recall: 99.74%\n",
      "F1: 99.69%\n",
      "--------------------\n",
      "Results on test\n",
      "Accuracy: 64.62%\n",
      "Precision: 98.67%\n",
      "Recall: 29.64%\n",
      "F1: 45.59%\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "!python scores_and_plots.py --results-dir \"runs/05-12-2023_12:14:51-SubtaskA-monolingual-gpt2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cffada75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results on validation\n",
      "MAE: 22.77671\n",
      "--------------------\n",
      "Results on test\n",
      "MAE: 19.22574\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "!python scores_and_plots.py --results-dir \"runs/13-12-2023_18:14:18-SubtaskC-bert\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d765664b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results on validation\n",
      "Accuracy: 93.14%\n",
      "Precision: 87.84%\n",
      "Recall: 99.17%\n",
      "F1: 93.16%\n",
      "--------------------\n",
      "Results on test\n",
      "Accuracy: 81.92%\n",
      "Precision: 84.94%\n",
      "Recall: 77.60%\n",
      "F1: 81.10%\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "!python scores_and_plots.py --results-dir \"runs/13-12-2023_19:08:48-SubtaskA-monolingual-bert\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "48254a69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results on validation\n",
      "Accuracy: 92.83%\n",
      "Precision: 87.65%\n",
      "Recall: 98.67%\n",
      "F1: 92.84%\n",
      "--------------------\n",
      "Results on test\n",
      "Accuracy: 76.02%\n",
      "Precision: 83.00%\n",
      "Recall: 65.44%\n",
      "F1: 73.18%\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "!python scores_and_plots.py --results-dir \"runs/14-12-2023_01:45:56-SubtaskA-monolingual-bert\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}