step-3-reranking-best.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-17T11:10:46.276009Z",
     "iopub.status.busy": "2023-02-17T11:10:46.275558Z",
     "iopub.status.idle": "2023-02-17T11:10:52.662982Z",
     "shell.execute_reply": "2023-02-17T11:10:52.662029Z",
     "shell.execute_reply.started": "2023-02-17T11:10:46.275925Z"
    }
   },
   "outputs": [],
   "source": [
    "# =========================================================================================\n",
    "# Libraries\n",
    "# =========================================================================================\n",
    "import os\n",
    "import gc\n",
    "import time\n",
    "import math\n",
    "import random\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.optim import AdamW\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torch.utils.checkpoint import checkpoint\n",
    "import tokenizers\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
    "from transformers import get_cosine_schedule_with_warmup, DataCollatorWithPadding\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "#%env TOKENIZERS_PARALLELISM=true\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"max_split_size_mb:21\"\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n",
    "# =========================================================================================\n",
    "# Configurations\n",
    "# =========================================================================================\n",
    "class CFG:\n",
    "    print_freq = 1000\n",
    "    num_workers = 24\n",
    "    model = 'model/stage-1-paraphrase-multilingual-mpnet-base-v2-epochs-40-seq-50-tuned'\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model)\n",
    "    gradient_checkpointing = False\n",
    "    num_cycles = 0.5\n",
    "    warmup_ratio = 0.1\n",
    "    epochs = 7\n",
    "    encoder_lr = 1e-5\n",
    "    decoder_lr = 1e-4\n",
    "    eps = 1e-6\n",
    "    betas = (0.9, 0.999)\n",
    "    batch_size = 32\n",
    "    weight_decay = 0.01\n",
    "    max_grad_norm = 0.012\n",
    "    #max_len = 512\n",
    "    max_len = 128\n",
    "    n_folds = 5\n",
    "    seed = 42\n",
    "    \n",
    "# =========================================================================================\n",
    "# Seed everything for deterministic results\n",
    "# =========================================================================================\n",
    "def seed_everything(cfg):\n",
    "    random.seed(cfg.seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(cfg.seed)\n",
    "    np.random.seed(cfg.seed)\n",
    "    torch.manual_seed(cfg.seed)\n",
    "    torch.cuda.manual_seed(cfg.seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "# =========================================================================================\n",
    "# F2 score metric\n",
    "# =========================================================================================\n",
    "def f2_score(y_true, y_pred):\n",
    "    y_true = y_true.apply(lambda x: set(x.split()))\n",
    "    y_pred = y_pred.apply(lambda x: set(x.split()))\n",
    "    tp = np.array([len(x[0] & x[1]) for x in zip(y_true, y_pred)])\n",
    "    fp = np.array([len(x[1] - x[0]) for x in zip(y_true, y_pred)])\n",
    "    fn = np.array([len(x[0] - x[1]) for x in zip(y_true, y_pred)])\n",
    "    precision = tp / (tp + fp)\n",
    "    recall = tp / (tp + fn)\n",
    "    f2 = tp / (tp + 0.2 * fp + 0.8 * fn)\n",
    "    return round(f2.mean(), 4)\n",
    "\n",
    "# =========================================================================================\n",
    "# Data Loading\n",
    "# =========================================================================================\n",
    "def read_data(cfg):\n",
    "    train = pd.read_parquet('data/candidates_25.parquet')\n",
    "    train['title1'].fillna(\"no title\", inplace = True)\n",
    "    train['title2'].fillna(\"no title\", inplace = True)\n",
    "    #topics['description'].fillna(\"no description\", inplace = True)\n",
    "    #content['description'].fillna(\"no description\", inplace = True)\n",
    "    \n",
    "    correlations = pd.read_csv('data/correlation_folds.csv')\n",
    "    \n",
    "    # Create feature column\n",
    "    train['text'] = train['title1'] + '[SEP]' + train['title2']\n",
    "    print(' ')\n",
    "    print('-' * 50)\n",
    "    print(f\"train.shape: {train.shape}\")\n",
    "    print(f\"correlations.shape: {correlations.shape}\")\n",
    "    return train, correlations\n",
    "\n",
    "\n",
    "# =========================================================================================\n",
    "# Get max length\n",
    "# =========================================================================================\n",
    "def get_max_length(train, cfg):\n",
    "    lengths = []\n",
    "    for text in tqdm(train['text'].fillna(\"\").values, total = len(train)):\n",
    "        length = len(cfg.tokenizer(text, add_special_tokens = False)['input_ids'])\n",
    "        lengths.append(length)\n",
    "    cfg.max_len = max(lengths) + 2 # cls & sep\n",
    "    print(f\"max_len: {cfg.max_len}\")\n",
    "\n",
    "# =========================================================================================\n",
    "# Prepare input, tokenize\n",
    "# =========================================================================================\n",
    "def prepare_input(text, cfg):\n",
    "    inputs = cfg.tokenizer.encode_plus(\n",
    "        text, \n",
    "        return_tensors = None, \n",
    "        add_special_tokens = True, \n",
    "        max_length = cfg.max_len,\n",
    "        pad_to_max_length = True,\n",
    "        truncation = True\n",
    "    )\n",
    "    for k, v in inputs.items():\n",
    "        inputs[k] = torch.tensor(v, dtype = torch.long)\n",
    "    return inputs\n",
    "\n",
    "# =========================================================================================\n",
    "# Custom dataset\n",
    "# =========================================================================================\n",
    "class custom_dataset(Dataset):\n",
    "    def __init__(self, df, cfg):\n",
    "        self.cfg = cfg\n",
    "        self.texts = df['text'].values\n",
    "        self.labels = df['target'].values\n",
    "    def __len__(self):\n",
    "        return len(self.texts)\n",
    "    def __getitem__(self, item):\n",
    "        inputs = prepare_input(self.texts[item], self.cfg)\n",
    "        label = torch.tensor(self.labels[item], dtype = torch.float)\n",
    "        return inputs, label\n",
    "    \n",
    "# =========================================================================================\n",
    "# Collate function for training\n",
    "# =========================================================================================\n",
    "def collate(inputs):\n",
    "    mask_len = int(inputs[\"attention_mask\"].sum(axis=1).max())\n",
    "    for k, v in inputs.items():\n",
    "        inputs[k] = inputs[k][:,:mask_len]\n",
    "    return inputs\n",
    "\n",
    "# =========================================================================================\n",
    "# Mean pooling class\n",
    "# =========================================================================================\n",
    "class MeanPooling(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MeanPooling, self).__init__()\n",
    "    def forward(self, last_hidden_state, attention_mask):\n",
    "        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n",
    "        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n",
    "        sum_mask = input_mask_expanded.sum(1)\n",
    "        sum_mask = torch.clamp(sum_mask, min=1e-9)\n",
    "        mean_embeddings = sum_embeddings / sum_mask\n",
    "        return mean_embeddings\n",
    "    \n",
    "# =========================================================================================\n",
    "# Model\n",
    "# =========================================================================================\n",
    "class custom_model(nn.Module):\n",
    "    def __init__(self, cfg):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg\n",
    "        self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states = True)\n",
    "        self.config.hidden_dropout = 0.0\n",
    "        self.config.hidden_dropout_prob = 0.0\n",
    "        self.config.attention_dropout = 0.0\n",
    "        self.config.attention_probs_dropout_prob = 0.0\n",
    "        self.model = AutoModel.from_pretrained(cfg.model, config = self.config)\n",
    "        if self.cfg.gradient_checkpointing:\n",
    "            self.model.gradient_checkpointing_enable()\n",
    "        self.pool = MeanPooling()\n",
    "        self.fc = nn.Linear(self.config.hidden_size, 1)\n",
    "        self._init_weights(self.fc)\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "    def feature(self, inputs):\n",
    "        outputs = self.model(**inputs)\n",
    "        last_hidden_state = outputs.last_hidden_state\n",
    "        feature = self.pool(last_hidden_state, inputs['attention_mask'])\n",
    "        return feature\n",
    "    def forward(self, inputs):\n",
    "        feature = self.feature(inputs)\n",
    "        output = self.fc(feature)\n",
    "        return output\n",
    "    \n",
    "# =========================================================================================\n",
    "# Helper functions\n",
    "# =========================================================================================\n",
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count\n",
    "\n",
    "def asMinutes(s):\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)\n",
    "\n",
    "def timeSince(since, percent):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    es = s / (percent)\n",
    "    rs = es - s\n",
    "    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n",
    "\n",
    "# =========================================================================================\n",
    "# Train function loop\n",
    "# =========================================================================================\n",
    "def train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device, cfg):\n",
    "    model.train()\n",
    "    scaler = torch.cuda.amp.GradScaler(enabled = True)\n",
    "    losses = AverageMeter()\n",
    "    start = end = time.time()\n",
    "    global_step = 0\n",
    "    for step, (inputs, target) in enumerate(train_loader):\n",
    "        inputs = collate(inputs)\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = v.to(device)\n",
    "        target = target.to(device)\n",
    "        batch_size = target.size(0)\n",
    "        with torch.cuda.amp.autocast(enabled = True):\n",
    "            y_preds = model(inputs)\n",
    "            loss = criterion(y_preds.view(-1), target)\n",
    "        losses.update(loss.item(), batch_size)\n",
    "        scaler.scale(loss).backward()\n",
    "        scaler.unscale_(optimizer)\n",
    "        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.max_grad_norm)\n",
    "        scaler.step(optimizer)\n",
    "        scaler.update()\n",
    "        optimizer.zero_grad()\n",
    "        global_step += 1\n",
    "        scheduler.step()\n",
    "        end = time.time()\n",
    "        if step % cfg.print_freq == 0 or step == (len(train_loader) - 1):\n",
    "            print('Epoch: [{0}][{1}/{2}] '\n",
    "                  'Elapsed {remain:s} '\n",
    "                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n",
    "                  'Grad: {grad_norm:.4f}  '\n",
    "                  'LR: {lr:.8f}  '\n",
    "                  .format(epoch + 1, \n",
    "                          step, \n",
    "                          len(train_loader), \n",
    "                          remain = timeSince(start, float(step + 1) / len(train_loader)),\n",
    "                          loss = losses,\n",
    "                          grad_norm = grad_norm,\n",
    "                          lr = scheduler.get_lr()[0]))\n",
    "    return losses.avg\n",
    "\n",
    "# =========================================================================================\n",
    "# Valid function loop\n",
    "# =========================================================================================\n",
    "def valid_fn(valid_loader, model, criterion, device, cfg):\n",
    "    losses = AverageMeter()\n",
    "    model.eval()\n",
    "    preds = []\n",
    "    start = end = time.time()\n",
    "    for step, (inputs, target) in enumerate(valid_loader):\n",
    "        inputs = collate(inputs)\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = v.to(device)\n",
    "        target = target.to(device)\n",
    "        batch_size = target.size(0)\n",
    "        with torch.no_grad():\n",
    "            y_preds = model(inputs)\n",
    "        loss = criterion(y_preds.view(-1), target)\n",
    "        losses.update(loss.item(), batch_size)\n",
    "        preds.append(y_preds.sigmoid().squeeze().to('cpu').numpy().reshape(-1))\n",
    "        end = time.time()\n",
    "        if step % cfg.print_freq == 0 or step == (len(valid_loader) - 1):\n",
    "            print('EVAL: [{0}/{1}] '\n",
    "                  'Elapsed {remain:s} '\n",
    "                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n",
    "                  .format(step, \n",
    "                          len(valid_loader),\n",
    "                          loss = losses,\n",
    "                          remain = timeSince(start, float(step + 1) / len(valid_loader))))\n",
    "    predictions = np.concatenate(preds, axis = 0)\n",
    "    return losses.avg, predictions\n",
    "\n",
    "# =========================================================================================\n",
    "# Get best threshold\n",
    "# =========================================================================================\n",
    "def get_best_threshold(x_val, val_predictions, correlations):\n",
    "    best_score = 0\n",
    "    best_threshold = None\n",
    "    for thres in np.arange(0.0001, 0.99, 0.001):\n",
    "        x_val['predictions'] = np.where(val_predictions > thres, 1, 0)\n",
    "        x_val1 = x_val[x_val['predictions'] == 1]\n",
    "        x_val1 = x_val1.groupby(['topics_ids'])['content_ids'].unique().reset_index()\n",
    "        x_val1['content_ids'] = x_val1['content_ids'].apply(lambda x: ' '.join(x))\n",
    "        x_val1.columns = ['topic_id', 'predictions']\n",
    "        x_val0 = pd.Series(x_val['topics_ids'].unique())\n",
    "        x_val0 = x_val0[~x_val0.isin(x_val1['topic_id'])]\n",
    "        x_val0 = pd.DataFrame({'topic_id': x_val0.values, 'predictions': \"\"})\n",
    "        x_val_r = pd.concat([x_val1, x_val0], axis = 0, ignore_index = True)\n",
    "        x_val_r = x_val_r.merge(correlations, how = 'left', on = 'topic_id')\n",
    "        score = f2_score(x_val_r['content_ids'], x_val_r['predictions'])\n",
    "        if score > best_score:\n",
    "            best_score = score\n",
    "            best_threshold = thres\n",
    "    return best_score, best_threshold\n",
    "    \n",
    "# =========================================================================================\n",
    "# Train & Evaluate\n",
    "# =========================================================================================\n",
    "def train_and_evaluate_one_fold(train, correlations, fold, cfg):\n",
    "    print(' ')\n",
    "    print(f\"========== fold: {fold} training ==========\")\n",
    "    # Split train & validation\n",
    "    x_train = train[train['fold'] != fold]\n",
    "    #pos = x_train[x_train.target==1]\n",
    "    #neg = x_train[x_train.target==0].groupby('topics_ids').sample(frac=0.4)\n",
    "    #x_train = pd.concat([pos,neg],axis=0).reset_index(drop=True)\n",
    "    #x_train = x_train[x_train.topics_ids.isin(x_train.groupby(['topics_ids']).sample(1).sample(frac=0.5,random_state=1).topics_ids.unique())]\n",
    "    x_val = train[train['fold'] == fold]\n",
    "    valid_labels = x_val['target'].values\n",
    "    train_dataset = custom_dataset(x_train, cfg)\n",
    "    valid_dataset = custom_dataset(x_val, cfg)\n",
    "    train_loader = DataLoader(\n",
    "        train_dataset, \n",
    "        batch_size = cfg.batch_size, \n",
    "        shuffle = True, \n",
    "        num_workers = cfg.num_workers, \n",
    "        pin_memory = True, \n",
    "        drop_last = True\n",
    "    )\n",
    "    valid_loader = DataLoader(\n",
    "        valid_dataset, \n",
    "        batch_size = cfg.batch_size, \n",
    "        shuffle = False, \n",
    "        num_workers = cfg.num_workers, \n",
    "        pin_memory = True, \n",
    "        drop_last = False\n",
    "    )\n",
    "    # Get model\n",
    "    model = custom_model(cfg)\n",
    "    model.to(device)\n",
    "    # Optimizer\n",
    "    def get_optimizer_params(model, encoder_lr, decoder_lr, weight_decay = 0.0):\n",
    "        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "        optimizer_parameters = [\n",
    "            {'params': [p for n, p in model.model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "            'lr': encoder_lr, 'weight_decay': weight_decay},\n",
    "            {'params': [p for n, p in model.model.named_parameters() if any(nd in n for nd in no_decay)],\n",
    "            'lr': encoder_lr, 'weight_decay': 0.0},\n",
    "            {'params': [p for n, p in model.named_parameters() if \"model\" not in n],\n",
    "            'lr': decoder_lr, 'weight_decay': 0.0}\n",
    "        ]\n",
    "        return optimizer_parameters\n",
    "    optimizer_parameters = get_optimizer_params(\n",
    "        model, \n",
    "        encoder_lr = cfg.encoder_lr, \n",
    "        decoder_lr = cfg.decoder_lr,\n",
    "        weight_decay = cfg.weight_decay\n",
    "    )\n",
    "    optimizer = AdamW(\n",
    "        optimizer_parameters, \n",
    "        lr = cfg.encoder_lr, \n",
    "        eps = cfg.eps, \n",
    "        betas = cfg.betas\n",
    "    )\n",
    "    num_train_steps = int(len(x_train) / cfg.batch_size * cfg.epochs)\n",
    "    num_warmup_steps = num_train_steps * cfg.warmup_ratio\n",
    "    # Scheduler\n",
    "    scheduler = get_cosine_schedule_with_warmup(\n",
    "        optimizer, \n",
    "        num_warmup_steps = num_warmup_steps, \n",
    "        num_training_steps = num_train_steps, \n",
    "        num_cycles = cfg.num_cycles\n",
    "        )\n",
    "    # Training & Validation loop\n",
    "    criterion = nn.BCEWithLogitsLoss(reduction = \"mean\")\n",
    "    best_score = 0\n",
    "    for epoch in range(cfg.epochs):\n",
    "        start_time = time.time()\n",
    "        # Train\n",
    "        avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device, cfg)\n",
    "        # Validation\n",
    "        avg_val_loss, predictions = valid_fn(valid_loader, model, criterion, device, cfg)\n",
    "        # Compute f2_score\n",
    "        score, threshold = get_best_threshold(x_val, predictions, correlations)\n",
    "        elapsed = time.time() - start_time\n",
    "        print(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n",
    "        print(f'Epoch {epoch+1} - Score: {score:.4f} - Threshold: {threshold:.5f}')\n",
    "        if score > best_score:\n",
    "            best_score = score\n",
    "        print(f'Epoch {epoch+1} - Save: {score:.4f} Model')\n",
    "        torch.save(\n",
    "                {'model': model.state_dict(), 'predictions': predictions}, \n",
    "                f\"{cfg.model.replace('/', '-')}_fold{fold}_{cfg.seed}_20_{epoch}.pth\"\n",
    "                )\n",
    "        val_predictions = predictions\n",
    "    torch.cuda.empty_cache()\n",
    "    gc.collect()\n",
    "    # Get best threshold\n",
    "    best_score, best_threshold = get_best_threshold(x_val, val_predictions, correlations)\n",
    "    print(f'Our CV score is {best_score} using a threshold of {best_threshold}')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-17T11:10:52.666005Z",
     "iopub.status.busy": "2023-02-17T11:10:52.665003Z",
     "iopub.status.idle": "2023-02-17T11:10:56.708092Z",
     "shell.execute_reply": "2023-02-17T11:10:56.707004Z",
     "shell.execute_reply.started": "2023-02-17T11:10:52.665964Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "--------------------------------------------------\n",
      "train.shape: (1537925, 7)\n",
      "correlations.shape: (61517, 3)\n"
     ]
    }
   ],
   "source": [
    "# Seed everything\n",
    "seed_everything(CFG)\n",
    "# Read data\n",
    "train, correlations = read_data(CFG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get max length\n",
    "#get_max_length(train, CFG)\n",
    "#2321"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CFG.max_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-17T11:10:59.390099Z",
     "iopub.status.busy": "2023-02-17T11:10:59.389739Z",
     "iopub.status.idle": "2023-02-17T11:19:59.654542Z",
     "shell.execute_reply": "2023-02-17T11:19:59.652840Z",
     "shell.execute_reply.started": "2023-02-17T11:10:59.390068Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "========== fold: 0 training ==========\n",
      "Epoch: [1][0/38447] Elapsed 0m 2s (remain 1877m 40s) Loss: 0.6818(0.6818) Grad: 1.0100  LR: 0.00000000  \n",
      "Epoch: [1][1000/38447] Elapsed 2m 7s (remain 79m 15s) Loss: 0.4495(0.6082) Grad: 0.7898  LR: 0.00000037  \n",
      "Epoch: [1][2000/38447] Elapsed 4m 12s (remain 76m 40s) Loss: 0.2687(0.5177) Grad: 0.9810  LR: 0.00000074  \n",
      "Epoch: [1][3000/38447] Elapsed 6m 22s (remain 75m 19s) Loss: 0.3034(0.4859) Grad: 1.8262  LR: 0.00000112  \n",
      "Epoch: [1][4000/38447] Elapsed 8m 40s (remain 74m 36s) Loss: 0.3209(0.4670) Grad: 1.5145  LR: 0.00000149  \n",
      "Epoch: [1][5000/38447] Elapsed 10m 57s (remain 73m 16s) Loss: 0.3587(0.4566) Grad: 1.6685  LR: 0.00000186  \n",
      "Epoch: [1][6000/38447] Elapsed 13m 20s (remain 72m 8s) Loss: 0.4350(0.4487) Grad: 1.9599  LR: 0.00000223  \n",
      "Epoch: [1][7000/38447] Elapsed 15m 43s (remain 70m 37s) Loss: 0.2409(0.4406) Grad: 1.4282  LR: 0.00000260  \n",
      "Epoch: [1][8000/38447] Elapsed 18m 6s (remain 68m 54s) Loss: 0.3646(0.4342) Grad: 3.0226  LR: 0.00000297  \n",
      "Epoch: [1][9000/38447] Elapsed 20m 29s (remain 67m 2s) Loss: 0.4042(0.4289) Grad: 4.1542  LR: 0.00000334  \n",
      "Epoch: [1][10000/38447] Elapsed 22m 52s (remain 65m 3s) Loss: 0.2557(0.4241) Grad: 3.3363  LR: 0.00000372  \n",
      "Epoch: [1][11000/38447] Elapsed 25m 15s (remain 63m 0s) Loss: 0.5366(0.4198) Grad: 4.6721  LR: 0.00000409  \n",
      "Epoch: [1][12000/38447] Elapsed 27m 38s (remain 60m 54s) Loss: 0.3493(0.4153) Grad: 2.4862  LR: 0.00000446  \n",
      "Epoch: [1][13000/38447] Elapsed 29m 52s (remain 58m 28s) Loss: 0.5386(0.4119) Grad: 4.4658  LR: 0.00000483  \n",
      "Epoch: [1][14000/38447] Elapsed 31m 57s (remain 55m 47s) Loss: 0.2310(0.4084) Grad: 2.0798  LR: 0.00000520  \n",
      "Epoch: [1][15000/38447] Elapsed 34m 1s (remain 53m 11s) Loss: 0.2152(0.4051) Grad: 2.1335  LR: 0.00000557  \n",
      "Epoch: [1][16000/38447] Elapsed 36m 5s (remain 50m 38s) Loss: 0.4087(0.4017) Grad: 3.4564  LR: 0.00000595  \n",
      "Epoch: [1][17000/38447] Elapsed 38m 10s (remain 48m 8s) Loss: 0.3779(0.3986) Grad: 3.5405  LR: 0.00000632  \n",
      "Epoch: [1][18000/38447] Elapsed 40m 14s (remain 45m 42s) Loss: 0.3643(0.3957) Grad: 2.4013  LR: 0.00000669  \n",
      "Epoch: [1][19000/38447] Elapsed 42m 18s (remain 43m 18s) Loss: 0.4786(0.3929) Grad: 3.9216  LR: 0.00000706  \n",
      "Epoch: [1][20000/38447] Elapsed 44m 24s (remain 40m 57s) Loss: 0.2269(0.3903) Grad: 3.9262  LR: 0.00000743  \n",
      "Epoch: [1][21000/38447] Elapsed 46m 28s (remain 38m 36s) Loss: 0.4214(0.3880) Grad: 3.9153  LR: 0.00000780  \n",
      "Epoch: [1][22000/38447] Elapsed 48m 32s (remain 36m 17s) Loss: 0.4557(0.3856) Grad: 5.2311  LR: 0.00000817  \n",
      "Epoch: [1][23000/38447] Elapsed 50m 37s (remain 33m 59s) Loss: 0.5900(0.3834) Grad: 3.9390  LR: 0.00000855  \n",
      "Epoch: [1][24000/38447] Elapsed 52m 41s (remain 31m 42s) Loss: 0.3427(0.3811) Grad: 2.7239  LR: 0.00000892  \n",
      "Epoch: [1][25000/38447] Elapsed 54m 45s (remain 29m 27s) Loss: 0.5119(0.3789) Grad: 4.7100  LR: 0.00000929  \n",
      "Epoch: [1][26000/38447] Elapsed 56m 50s (remain 27m 12s) Loss: 0.3206(0.3765) Grad: 3.2696  LR: 0.00000966  \n",
      "Epoch: [1][27000/38447] Elapsed 58m 54s (remain 24m 58s) Loss: 0.5139(0.3743) Grad: 3.8862  LR: 0.00001000  \n",
      "Epoch: [1][28000/38447] Elapsed 60m 58s (remain 22m 44s) Loss: 0.1644(0.3724) Grad: 2.6066  LR: 0.00001000  \n",
      "Epoch: [1][29000/38447] Elapsed 63m 2s (remain 20m 32s) Loss: 0.3916(0.3705) Grad: 4.4863  LR: 0.00001000  \n",
      "Epoch: [1][30000/38447] Elapsed 65m 7s (remain 18m 19s) Loss: 0.2518(0.3686) Grad: 2.8663  LR: 0.00001000  \n",
      "Epoch: [1][31000/38447] Elapsed 67m 11s (remain 16m 8s) Loss: 0.2798(0.3666) Grad: 3.1309  LR: 0.00000999  \n",
      "Epoch: [1][32000/38447] Elapsed 69m 14s (remain 13m 56s) Loss: 0.3554(0.3649) Grad: 4.1739  LR: 0.00000999  \n",
      "Epoch: [1][33000/38447] Elapsed 71m 18s (remain 11m 46s) Loss: 0.2229(0.3631) Grad: 3.5234  LR: 0.00000998  \n",
      "Epoch: [1][34000/38447] Elapsed 73m 21s (remain 9m 35s) Loss: 0.5404(0.3613) Grad: 6.6193  LR: 0.00000998  \n",
      "Epoch: [1][35000/38447] Elapsed 75m 25s (remain 7m 25s) Loss: 0.5873(0.3598) Grad: 4.7244  LR: 0.00000997  \n",
      "Epoch: [1][36000/38447] Elapsed 77m 28s (remain 5m 15s) Loss: 0.2365(0.3581) Grad: 2.6368  LR: 0.00000997  \n",
      "Epoch: [1][37000/38447] Elapsed 79m 32s (remain 3m 6s) Loss: 0.3292(0.3565) Grad: 6.7953  LR: 0.00000996  \n",
      "Epoch: [1][38000/38447] Elapsed 81m 35s (remain 0m 57s) Loss: 0.1805(0.3550) Grad: 3.2660  LR: 0.00000995  \n",
      "Epoch: [1][38446/38447] Elapsed 82m 30s (remain 0m 0s) Loss: 0.2715(0.3543) Grad: 3.0713  LR: 0.00000994  \n",
      "EVAL: [0/9613] Elapsed 0m 1s (remain 312m 17s) Loss: 1.0917(1.0917) \n",
      "EVAL: [1000/9613] Elapsed 0m 49s (remain 7m 2s) Loss: 0.2813(0.1721) \n",
      "EVAL: [2000/9613] Elapsed 1m 41s (remain 6m 27s) Loss: 0.3177(0.2031) \n",
      "EVAL: [3000/9613] Elapsed 2m 32s (remain 5m 36s) Loss: 0.2018(0.2092) \n",
      "EVAL: [4000/9613] Elapsed 3m 26s (remain 4m 49s) Loss: 0.1971(0.2197) \n",
      "EVAL: [5000/9613] Elapsed 4m 20s (remain 3m 59s) Loss: 0.0543(0.2260) \n",
      "EVAL: [6000/9613] Elapsed 5m 14s (remain 3m 9s) Loss: 0.3162(0.2326) \n",
      "EVAL: [7000/9613] Elapsed 6m 9s (remain 2m 17s) Loss: 0.3054(0.2390) \n",
      "EVAL: [8000/9613] Elapsed 7m 3s (remain 1m 25s) Loss: 0.2678(0.2474) \n",
      "EVAL: [9000/9613] Elapsed 7m 57s (remain 0m 32s) Loss: 0.1885(0.2567) \n",
      "EVAL: [9612/9613] Elapsed 8m 30s (remain 0m 0s) Loss: 0.9093(0.2648) \n",
      "Epoch 1 - avg_train_loss: 0.3543  avg_val_loss: 0.2648  time: 5683s\n",
      "Epoch 1 - Score: 0.4797 - Threshold: 0.07810\n",
      "Epoch 1 - Save: 0.4797 Model\n",
      "Epoch: [2][0/38447] Elapsed 0m 2s (remain 1416m 58s) Loss: 0.2374(0.2374) Grad: 4.3647  LR: 0.00000994  \n",
      "Epoch: [2][1000/38447] Elapsed 2m 5s (remain 78m 9s) Loss: 0.1525(0.2755) Grad: 3.0476  LR: 0.00000993  \n",
      "Epoch: [2][2000/38447] Elapsed 4m 8s (remain 75m 32s) Loss: 0.2450(0.2775) Grad: 4.3877  LR: 0.00000992  \n",
      "Epoch: [2][3000/38447] Elapsed 6m 12s (remain 73m 16s) Loss: 0.1176(0.2759) Grad: 2.8576  LR: 0.00000991  \n",
      "Epoch: [2][4000/38447] Elapsed 8m 15s (remain 71m 7s) Loss: 0.2264(0.2763) Grad: 3.5518  LR: 0.00000990  \n",
      "Epoch: [2][5000/38447] Elapsed 10m 21s (remain 69m 17s) Loss: 0.1213(0.2766) Grad: 3.6681  LR: 0.00000989  \n",
      "Epoch: [2][6000/38447] Elapsed 12m 25s (remain 67m 13s) Loss: 0.2595(0.2759) Grad: 4.4566  LR: 0.00000987  \n",
      "Epoch: [2][7000/38447] Elapsed 14m 31s (remain 65m 16s) Loss: 0.2749(0.2749) Grad: 3.3468  LR: 0.00000986  \n",
      "Epoch: [2][8000/38447] Elapsed 16m 35s (remain 63m 7s) Loss: 0.1388(0.2745) Grad: 2.4869  LR: 0.00000984  \n",
      "Epoch: [2][9000/38447] Elapsed 18m 38s (remain 61m 0s) Loss: 0.1851(0.2739) Grad: 4.6631  LR: 0.00000982  \n",
      "Epoch: [2][10000/38447] Elapsed 20m 42s (remain 58m 53s) Loss: 0.0539(0.2731) Grad: 1.4325  LR: 0.00000981  \n",
      "Epoch: [2][11000/38447] Elapsed 22m 45s (remain 56m 47s) Loss: 0.4077(0.2721) Grad: 4.8425  LR: 0.00000979  \n",
      "Epoch: [2][12000/38447] Elapsed 24m 49s (remain 54m 42s) Loss: 0.2171(0.2720) Grad: 5.8763  LR: 0.00000977  \n",
      "Epoch: [2][13000/38447] Elapsed 26m 52s (remain 52m 36s) Loss: 0.1867(0.2713) Grad: 2.8856  LR: 0.00000975  \n",
      "Epoch: [2][14000/38447] Elapsed 28m 56s (remain 50m 31s) Loss: 0.5348(0.2706) Grad: 7.3625  LR: 0.00000973  \n",
      "Epoch: [2][15000/38447] Elapsed 30m 59s (remain 48m 26s) Loss: 0.2365(0.2703) Grad: 3.2683  LR: 0.00000971  \n",
      "Epoch: [2][16000/38447] Elapsed 33m 3s (remain 46m 22s) Loss: 0.3168(0.2701) Grad: 4.3671  LR: 0.00000968  \n",
      "Epoch: [2][17000/38447] Elapsed 35m 6s (remain 44m 17s) Loss: 0.2678(0.2700) Grad: 5.1541  LR: 0.00000966  \n",
      "Epoch: [2][18000/38447] Elapsed 37m 10s (remain 42m 13s) Loss: 0.1069(0.2698) Grad: 5.0330  LR: 0.00000964  \n",
      "Epoch: [2][19000/38447] Elapsed 39m 13s (remain 40m 9s) Loss: 0.2646(0.2691) Grad: 6.7767  LR: 0.00000961  \n",
      "Epoch: [2][20000/38447] Elapsed 41m 17s (remain 38m 4s) Loss: 0.3352(0.2689) Grad: 3.8326  LR: 0.00000959  \n",
      "Epoch: [2][21000/38447] Elapsed 43m 20s (remain 36m 0s) Loss: 0.1860(0.2685) Grad: 4.6585  LR: 0.00000956  \n",
      "Epoch: [2][22000/38447] Elapsed 45m 24s (remain 33m 56s) Loss: 0.1923(0.2684) Grad: 1.5881  LR: 0.00000953  \n",
      "Epoch: [2][23000/38447] Elapsed 47m 27s (remain 31m 52s) Loss: 0.1214(0.2677) Grad: 2.3495  LR: 0.00000951  \n",
      "Epoch: [2][24000/38447] Elapsed 49m 31s (remain 29m 48s) Loss: 0.2515(0.2671) Grad: 3.4662  LR: 0.00000948  \n",
      "Epoch: [2][25000/38447] Elapsed 51m 34s (remain 27m 44s) Loss: 0.2996(0.2666) Grad: 3.5063  LR: 0.00000945  \n",
      "Epoch: [2][26000/38447] Elapsed 53m 38s (remain 25m 40s) Loss: 0.4387(0.2662) Grad: 5.4531  LR: 0.00000942  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [2][27000/38447] Elapsed 55m 42s (remain 23m 36s) Loss: 0.2144(0.2656) Grad: 3.8990  LR: 0.00000939  \n",
      "Epoch: [2][28000/38447] Elapsed 57m 45s (remain 21m 32s) Loss: 0.2003(0.2650) Grad: 4.0993  LR: 0.00000936  \n",
      "Epoch: [2][29000/38447] Elapsed 59m 49s (remain 19m 29s) Loss: 0.3444(0.2646) Grad: 4.5430  LR: 0.00000932  \n",
      "Epoch: [2][30000/38447] Elapsed 61m 52s (remain 17m 25s) Loss: 0.0779(0.2640) Grad: 2.4535  LR: 0.00000929  \n",
      "Epoch: [2][31000/38447] Elapsed 63m 55s (remain 15m 21s) Loss: 0.1139(0.2635) Grad: 2.4088  LR: 0.00000926  \n",
      "Epoch: [2][32000/38447] Elapsed 65m 59s (remain 13m 17s) Loss: 0.3600(0.2630) Grad: 4.7007  LR: 0.00000922  \n",
      "Epoch: [2][33000/38447] Elapsed 68m 2s (remain 11m 13s) Loss: 0.4316(0.2625) Grad: 6.1413  LR: 0.00000919  \n",
      "Epoch: [2][34000/38447] Elapsed 70m 6s (remain 9m 10s) Loss: 0.2379(0.2620) Grad: 3.8547  LR: 0.00000915  \n",
      "Epoch: [2][35000/38447] Elapsed 72m 9s (remain 7m 6s) Loss: 0.2124(0.2614) Grad: 5.0433  LR: 0.00000912  \n",
      "Epoch: [2][36000/38447] Elapsed 74m 13s (remain 5m 2s) Loss: 0.4610(0.2610) Grad: 8.9984  LR: 0.00000908  \n",
      "Epoch: [2][37000/38447] Elapsed 76m 16s (remain 2m 58s) Loss: 0.2909(0.2606) Grad: 6.1617  LR: 0.00000904  \n",
      "Epoch: [2][38000/38447] Elapsed 78m 27s (remain 0m 55s) Loss: 0.2176(0.2603) Grad: 4.0791  LR: 0.00000900  \n",
      "Epoch: [2][38446/38447] Elapsed 79m 31s (remain 0m 0s) Loss: 0.2392(0.2601) Grad: 7.1315  LR: 0.00000899  \n",
      "EVAL: [0/9613] Elapsed 0m 2s (remain 377m 21s) Loss: 1.0705(1.0705) \n",
      "EVAL: [1000/9613] Elapsed 0m 58s (remain 8m 22s) Loss: 0.2624(0.1535) \n",
      "EVAL: [2000/9613] Elapsed 1m 59s (remain 7m 34s) Loss: 0.2093(0.1822) \n",
      "EVAL: [3000/9613] Elapsed 2m 58s (remain 6m 32s) Loss: 0.0879(0.1871) \n",
      "EVAL: [4000/9613] Elapsed 4m 1s (remain 5m 38s) Loss: 0.2064(0.1963) \n",
      "EVAL: [5000/9613] Elapsed 5m 1s (remain 4m 37s) Loss: 0.0466(0.2033) \n",
      "EVAL: [6000/9613] Elapsed 6m 9s (remain 3m 42s) Loss: 0.3247(0.2097) \n",
      "EVAL: [7000/9613] Elapsed 7m 17s (remain 2m 43s) Loss: 0.3068(0.2156) \n",
      "EVAL: [8000/9613] Elapsed 8m 27s (remain 1m 42s) Loss: 0.3003(0.2239) \n",
      "EVAL: [9000/9613] Elapsed 9m 33s (remain 0m 39s) Loss: 0.1761(0.2334) \n",
      "EVAL: [9612/9613] Elapsed 10m 14s (remain 0m 0s) Loss: 0.8252(0.2426) \n",
      "Epoch 2 - avg_train_loss: 0.2601  avg_val_loss: 0.2426  time: 5643s\n",
      "Epoch 2 - Score: 0.5221 - Threshold: 0.08610\n",
      "Epoch 2 - Save: 0.5221 Model\n",
      "Epoch: [3][0/38447] Elapsed 0m 3s (remain 1964m 9s) Loss: 0.1637(0.1637) Grad: 2.9367  LR: 0.00000899  \n",
      "Epoch: [3][1000/38447] Elapsed 2m 27s (remain 92m 3s) Loss: 0.1081(0.2100) Grad: 1.9028  LR: 0.00000895  \n",
      "Epoch: [3][2000/38447] Elapsed 4m 57s (remain 90m 22s) Loss: 0.3473(0.2078) Grad: 5.4869  LR: 0.00000891  \n",
      "Epoch: [3][3000/38447] Elapsed 7m 29s (remain 88m 31s) Loss: 0.0690(0.2086) Grad: 2.2583  LR: 0.00000887  \n",
      "Epoch: [3][4000/38447] Elapsed 10m 28s (remain 90m 10s) Loss: 0.2303(0.2085) Grad: 4.9994  LR: 0.00000882  \n",
      "Epoch: [3][5000/38447] Elapsed 13m 25s (remain 89m 44s) Loss: 0.2140(0.2080) Grad: 6.9637  LR: 0.00000878  \n",
      "Epoch: [3][6000/38447] Elapsed 16m 17s (remain 88m 4s) Loss: 0.3560(0.2076) Grad: 5.0581  LR: 0.00000874  \n",
      "Epoch: [3][7000/38447] Elapsed 18m 59s (remain 85m 15s) Loss: 0.1303(0.2084) Grad: 3.2692  LR: 0.00000870  \n",
      "Epoch: [3][8000/38447] Elapsed 22m 6s (remain 84m 5s) Loss: 0.1719(0.2087) Grad: 5.0789  LR: 0.00000865  \n",
      "Epoch: [3][9000/38447] Elapsed 25m 4s (remain 82m 2s) Loss: 0.0878(0.2081) Grad: 2.9949  LR: 0.00000861  \n",
      "Epoch: [3][10000/38447] Elapsed 27m 9s (remain 77m 14s) Loss: 0.1626(0.2079) Grad: 2.6477  LR: 0.00000856  \n",
      "Epoch: [3][11000/38447] Elapsed 29m 17s (remain 73m 5s) Loss: 0.2259(0.2078) Grad: 6.5698  LR: 0.00000852  \n",
      "Epoch: [3][12000/38447] Elapsed 31m 23s (remain 69m 10s) Loss: 0.4612(0.2078) Grad: 16.6732  LR: 0.00000847  \n",
      "Epoch: [3][13000/38447] Elapsed 33m 27s (remain 65m 29s) Loss: 0.1169(0.2077) Grad: 4.1309  LR: 0.00000842  \n",
      "Epoch: [3][14000/38447] Elapsed 35m 32s (remain 62m 3s) Loss: 0.1070(0.2077) Grad: 3.6334  LR: 0.00000837  \n",
      "Epoch: [3][15000/38447] Elapsed 37m 37s (remain 58m 47s) Loss: 0.3221(0.2074) Grad: 3.3166  LR: 0.00000833  \n",
      "Epoch: [3][16000/38447] Elapsed 39m 41s (remain 55m 41s) Loss: 0.1090(0.2074) Grad: 5.9926  LR: 0.00000828  \n",
      "Epoch: [3][17000/38447] Elapsed 41m 46s (remain 52m 41s) Loss: 0.1823(0.2076) Grad: 6.3432  LR: 0.00000823  \n",
      "Epoch: [3][18000/38447] Elapsed 43m 51s (remain 49m 48s) Loss: 0.1867(0.2078) Grad: 5.2633  LR: 0.00000818  \n",
      "Epoch: [3][19000/38447] Elapsed 45m 55s (remain 47m 0s) Loss: 0.1476(0.2077) Grad: 3.3150  LR: 0.00000813  \n",
      "Epoch: [3][20000/38447] Elapsed 48m 0s (remain 44m 16s) Loss: 0.0959(0.2077) Grad: 2.1473  LR: 0.00000808  \n",
      "Epoch: [3][21000/38447] Elapsed 50m 5s (remain 41m 36s) Loss: 0.0652(0.2077) Grad: 1.9966  LR: 0.00000803  \n",
      "Epoch: [3][22000/38447] Elapsed 52m 9s (remain 38m 59s) Loss: 0.1450(0.2078) Grad: 5.7608  LR: 0.00000797  \n",
      "Epoch: [3][23000/38447] Elapsed 54m 14s (remain 36m 25s) Loss: 0.0870(0.2076) Grad: 4.4123  LR: 0.00000792  \n",
      "Epoch: [3][24000/38447] Elapsed 56m 19s (remain 33m 54s) Loss: 0.1426(0.2075) Grad: 6.3761  LR: 0.00000787  \n",
      "Epoch: [3][25000/38447] Elapsed 58m 22s (remain 31m 23s) Loss: 0.3413(0.2075) Grad: 6.4593  LR: 0.00000782  \n",
      "Epoch: [3][26000/38447] Elapsed 60m 26s (remain 28m 55s) Loss: 0.0951(0.2074) Grad: 2.3684  LR: 0.00000776  \n",
      "Epoch: [3][27000/38447] Elapsed 62m 29s (remain 26m 29s) Loss: 0.1159(0.2071) Grad: 6.2191  LR: 0.00000771  \n",
      "Epoch: [3][28000/38447] Elapsed 64m 33s (remain 24m 4s) Loss: 0.1173(0.2071) Grad: 4.6828  LR: 0.00000765  \n",
      "Epoch: [3][29000/38447] Elapsed 66m 36s (remain 21m 41s) Loss: 0.4899(0.2071) Grad: 10.0913  LR: 0.00000760  \n",
      "Epoch: [3][30000/38447] Elapsed 68m 41s (remain 19m 20s) Loss: 0.1005(0.2070) Grad: 4.1197  LR: 0.00000754  \n",
      "Epoch: [3][31000/38447] Elapsed 70m 45s (remain 16m 59s) Loss: 0.2681(0.2070) Grad: 5.0342  LR: 0.00000749  \n",
      "Epoch: [3][32000/38447] Elapsed 72m 49s (remain 14m 40s) Loss: 0.1258(0.2067) Grad: 3.8794  LR: 0.00000743  \n",
      "Epoch: [3][33000/38447] Elapsed 74m 53s (remain 12m 21s) Loss: 0.1292(0.2066) Grad: 3.8316  LR: 0.00000737  \n",
      "Epoch: [3][34000/38447] Elapsed 77m 21s (remain 10m 6s) Loss: 0.4156(0.2064) Grad: 8.4760  LR: 0.00000732  \n",
      "Epoch: [3][35000/38447] Elapsed 80m 1s (remain 7m 52s) Loss: 0.0764(0.2064) Grad: 1.1951  LR: 0.00000726  \n",
      "Epoch: [3][36000/38447] Elapsed 82m 41s (remain 5m 37s) Loss: 0.1127(0.2064) Grad: 4.9600  LR: 0.00000720  \n",
      "Epoch: [3][37000/38447] Elapsed 85m 22s (remain 3m 20s) Loss: 0.0873(0.2062) Grad: 3.0382  LR: 0.00000714  \n",
      "Epoch: [3][38000/38447] Elapsed 88m 3s (remain 1m 2s) Loss: 0.1076(0.2060) Grad: 3.5185  LR: 0.00000708  \n",
      "Epoch: [3][38446/38447] Elapsed 89m 15s (remain 0m 0s) Loss: 0.3174(0.2059) Grad: 3.7083  LR: 0.00000706  \n",
      "EVAL: [0/9613] Elapsed 0m 1s (remain 316m 3s) Loss: 1.2342(1.2342) \n",
      "EVAL: [1000/9613] Elapsed 0m 49s (remain 7m 1s) Loss: 0.2775(0.1460) \n",
      "EVAL: [2000/9613] Elapsed 1m 41s (remain 6m 27s) Loss: 0.1575(0.1757) \n",
      "EVAL: [3000/9613] Elapsed 2m 32s (remain 5m 36s) Loss: 0.0430(0.1814) \n",
      "EVAL: [4000/9613] Elapsed 3m 26s (remain 4m 49s) Loss: 0.1362(0.1906) \n",
      "EVAL: [5000/9613] Elapsed 4m 20s (remain 3m 59s) Loss: 0.0640(0.1981) \n",
      "EVAL: [6000/9613] Elapsed 5m 14s (remain 3m 9s) Loss: 0.3013(0.2044) \n",
      "EVAL: [7000/9613] Elapsed 6m 9s (remain 2m 17s) Loss: 0.3152(0.2106) \n",
      "EVAL: [8000/9613] Elapsed 7m 3s (remain 1m 25s) Loss: 0.1647(0.2201) \n",
      "EVAL: [9000/9613] Elapsed 7m 57s (remain 0m 32s) Loss: 0.1620(0.2310) \n",
      "EVAL: [9612/9613] Elapsed 8m 30s (remain 0m 0s) Loss: 0.9770(0.2416) \n",
      "Epoch 3 - avg_train_loss: 0.2059  avg_val_loss: 0.2416  time: 6122s\n",
      "Epoch 3 - Score: 0.5412 - Threshold: 0.06810\n",
      "Epoch 3 - Save: 0.5412 Model\n",
      "Epoch: [4][0/38447] Elapsed 0m 2s (remain 1410m 38s) Loss: 0.1612(0.1612) Grad: 5.1468  LR: 0.00000706  \n",
      "Epoch: [4][1000/38447] Elapsed 2m 41s (remain 100m 35s) Loss: 0.0561(0.1587) Grad: 3.6001  LR: 0.00000700  \n",
      "Epoch: [4][2000/38447] Elapsed 5m 20s (remain 97m 21s) Loss: 0.1081(0.1602) Grad: 12.3289  LR: 0.00000694  \n",
      "Epoch: [4][3000/38447] Elapsed 8m 0s (remain 94m 35s) Loss: 0.1202(0.1603) Grad: 6.1557  LR: 0.00000688  \n",
      "Epoch: [4][4000/38447] Elapsed 10m 40s (remain 91m 52s) Loss: 0.2733(0.1607) Grad: 9.1638  LR: 0.00000682  \n",
      "Epoch: [4][5000/38447] Elapsed 13m 20s (remain 89m 12s) Loss: 0.0805(0.1622) Grad: 2.6247  LR: 0.00000676  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [4][6000/38447] Elapsed 16m 0s (remain 86m 31s) Loss: 0.0996(0.1616) Grad: 1.4623  LR: 0.00000670  \n",
      "Epoch: [4][7000/38447] Elapsed 18m 39s (remain 83m 50s) Loss: 0.1591(0.1616) Grad: 5.0124  LR: 0.00000663  \n",
      "Epoch: [4][8000/38447] Elapsed 21m 19s (remain 81m 10s) Loss: 0.1686(0.1621) Grad: 10.1068  LR: 0.00000657  \n",
      "Epoch: [4][9000/38447] Elapsed 23m 59s (remain 78m 29s) Loss: 0.1427(0.1617) Grad: 7.6765  LR: 0.00000651  \n",
      "Epoch: [4][10000/38447] Elapsed 26m 39s (remain 75m 49s) Loss: 0.1857(0.1621) Grad: 8.4563  LR: 0.00000645  \n",
      "Epoch: [4][11000/38447] Elapsed 29m 19s (remain 73m 9s) Loss: 0.3753(0.1624) Grad: 14.0093  LR: 0.00000639  \n",
      "Epoch: [4][12000/38447] Elapsed 31m 59s (remain 70m 29s) Loss: 0.2497(0.1625) Grad: 4.3934  LR: 0.00000633  \n",
      "Epoch: [4][13000/38447] Elapsed 34m 39s (remain 67m 49s) Loss: 0.1068(0.1623) Grad: 6.2711  LR: 0.00000626  \n",
      "Epoch: [4][14000/38447] Elapsed 37m 18s (remain 65m 9s) Loss: 0.0394(0.1625) Grad: 2.9870  LR: 0.00000620  \n",
      "Epoch: [4][15000/38447] Elapsed 39m 58s (remain 62m 29s) Loss: 0.0898(0.1624) Grad: 3.7185  LR: 0.00000614  \n",
      "Epoch: [4][16000/38447] Elapsed 42m 38s (remain 59m 49s) Loss: 0.1168(0.1625) Grad: 3.8991  LR: 0.00000607  \n",
      "Epoch: [4][17000/38447] Elapsed 45m 18s (remain 57m 9s) Loss: 0.0934(0.1628) Grad: 2.4330  LR: 0.00000601  \n",
      "Epoch: [4][18000/38447] Elapsed 47m 58s (remain 54m 29s) Loss: 0.2114(0.1629) Grad: 5.6065  LR: 0.00000595  \n",
      "Epoch: [4][19000/38447] Elapsed 50m 38s (remain 51m 49s) Loss: 0.0751(0.1630) Grad: 4.7805  LR: 0.00000588  \n",
      "Epoch: [4][20000/38447] Elapsed 53m 18s (remain 49m 9s) Loss: 0.0604(0.1630) Grad: 6.6599  LR: 0.00000582  \n",
      "Epoch: [4][21000/38447] Elapsed 55m 58s (remain 46m 29s) Loss: 0.2031(0.1631) Grad: 14.1115  LR: 0.00000575  \n",
      "Epoch: [4][22000/38447] Elapsed 58m 38s (remain 43m 49s) Loss: 0.2733(0.1632) Grad: 4.4343  LR: 0.00000569  \n",
      "Epoch: [4][23000/38447] Elapsed 61m 18s (remain 41m 9s) Loss: 0.0418(0.1632) Grad: 1.9677  LR: 0.00000563  \n",
      "Epoch: [4][24000/38447] Elapsed 63m 57s (remain 38m 30s) Loss: 0.1513(0.1634) Grad: 9.3120  LR: 0.00000556  \n",
      "Epoch: [4][25000/38447] Elapsed 66m 40s (remain 35m 51s) Loss: 0.2405(0.1634) Grad: 6.7934  LR: 0.00000550  \n",
      "Epoch: [4][26000/38447] Elapsed 69m 24s (remain 33m 13s) Loss: 0.1912(0.1634) Grad: 6.6265  LR: 0.00000543  \n",
      "Epoch: [4][27000/38447] Elapsed 72m 4s (remain 30m 33s) Loss: 0.0988(0.1634) Grad: 2.6347  LR: 0.00000537  \n",
      "Epoch: [4][28000/38447] Elapsed 74m 45s (remain 27m 53s) Loss: 0.0899(0.1635) Grad: 3.4090  LR: 0.00000530  \n",
      "Epoch: [4][29000/38447] Elapsed 77m 26s (remain 25m 13s) Loss: 0.4062(0.1635) Grad: 7.4042  LR: 0.00000524  \n",
      "Epoch: [4][30000/38447] Elapsed 80m 10s (remain 22m 34s) Loss: 0.1288(0.1635) Grad: 4.4907  LR: 0.00000517  \n",
      "Epoch: [4][31000/38447] Elapsed 82m 54s (remain 19m 54s) Loss: 0.0963(0.1634) Grad: 5.8110  LR: 0.00000511  \n",
      "Epoch: [4][32000/38447] Elapsed 85m 38s (remain 17m 15s) Loss: 0.1686(0.1633) Grad: 3.1840  LR: 0.00000504  \n",
      "Epoch: [4][33000/38447] Elapsed 88m 22s (remain 14m 34s) Loss: 0.3174(0.1631) Grad: 15.1680  LR: 0.00000498  \n",
      "Epoch: [4][34000/38447] Elapsed 91m 2s (remain 11m 54s) Loss: 0.0705(0.1631) Grad: 3.1115  LR: 0.00000491  \n",
      "Epoch: [4][35000/38447] Elapsed 93m 42s (remain 9m 13s) Loss: 0.0301(0.1629) Grad: 1.5421  LR: 0.00000485  \n",
      "Epoch: [4][36000/38447] Elapsed 96m 22s (remain 6m 32s) Loss: 0.0458(0.1629) Grad: 2.4853  LR: 0.00000478  \n",
      "Epoch: [4][37000/38447] Elapsed 99m 2s (remain 3m 52s) Loss: 0.1212(0.1627) Grad: 2.9207  LR: 0.00000472  \n",
      "Epoch: [4][38000/38447] Elapsed 101m 42s (remain 1m 11s) Loss: 0.0248(0.1627) Grad: 1.7693  LR: 0.00000466  \n",
      "Epoch: [4][38446/38447] Elapsed 102m 53s (remain 0m 0s) Loss: 0.1119(0.1626) Grad: 3.8921  LR: 0.00000463  \n",
      "EVAL: [0/9613] Elapsed 0m 2s (remain 322m 40s) Loss: 2.1497(2.1497) \n",
      "EVAL: [1000/9613] Elapsed 0m 49s (remain 7m 2s) Loss: 0.2992(0.1556) \n",
      "EVAL: [2000/9613] Elapsed 1m 41s (remain 6m 27s) Loss: 0.1811(0.1905) \n",
      "EVAL: [3000/9613] Elapsed 2m 32s (remain 5m 36s) Loss: 0.0247(0.1976) \n",
      "EVAL: [4000/9613] Elapsed 3m 26s (remain 4m 49s) Loss: 0.1526(0.2085) \n",
      "EVAL: [5000/9613] Elapsed 4m 20s (remain 3m 59s) Loss: 0.0313(0.2169) \n",
      "EVAL: [6000/9613] Elapsed 5m 14s (remain 3m 9s) Loss: 0.4528(0.2235) \n",
      "EVAL: [7000/9613] Elapsed 6m 9s (remain 2m 17s) Loss: 0.4551(0.2328) \n",
      "EVAL: [8000/9613] Elapsed 7m 3s (remain 1m 25s) Loss: 0.2683(0.2445) \n",
      "EVAL: [9000/9613] Elapsed 7m 57s (remain 0m 32s) Loss: 0.0641(0.2589) \n",
      "EVAL: [9612/9613] Elapsed 8m 30s (remain 0m 0s) Loss: 0.9257(0.2735) \n",
      "Epoch 4 - avg_train_loss: 0.1626  avg_val_loss: 0.2735  time: 6937s\n",
      "Epoch 4 - Score: 0.5465 - Threshold: 0.03710\n",
      "Epoch 4 - Save: 0.5465 Model\n",
      "Epoch: [5][0/38447] Elapsed 0m 2s (remain 1438m 39s) Loss: 0.0876(0.0876) Grad: 3.2323  LR: 0.00000463  \n",
      "Epoch: [5][1000/38447] Elapsed 2m 42s (remain 101m 1s) Loss: 0.0957(0.1277) Grad: 11.3063  LR: 0.00000456  \n",
      "Epoch: [5][2000/38447] Elapsed 5m 22s (remain 97m 45s) Loss: 0.1492(0.1255) Grad: 10.0488  LR: 0.00000450  \n",
      "Epoch: [5][3000/38447] Elapsed 8m 6s (remain 95m 50s) Loss: 0.1558(0.1269) Grad: 36.0111  LR: 0.00000443  \n",
      "Epoch: [5][4000/38447] Elapsed 10m 47s (remain 92m 58s) Loss: 0.0350(0.1265) Grad: 1.5207  LR: 0.00000437  \n",
      "Epoch: [5][5000/38447] Elapsed 13m 28s (remain 90m 9s) Loss: 0.0544(0.1256) Grad: 5.7445  LR: 0.00000430  \n",
      "Epoch: [5][6000/38447] Elapsed 16m 9s (remain 87m 20s) Loss: 0.1296(0.1265) Grad: 8.0588  LR: 0.00000424  \n",
      "Epoch: [5][7000/38447] Elapsed 18m 49s (remain 84m 32s) Loss: 0.0799(0.1262) Grad: 2.5382  LR: 0.00000418  \n",
      "Epoch: [5][8000/38447] Elapsed 21m 29s (remain 81m 46s) Loss: 0.2927(0.1263) Grad: 4.2317  LR: 0.00000411  \n",
      "Epoch: [5][9000/38447] Elapsed 24m 9s (remain 79m 2s) Loss: 0.2491(0.1264) Grad: 6.7574  LR: 0.00000405  \n",
      "Epoch: [5][10000/38447] Elapsed 26m 49s (remain 76m 18s) Loss: 0.0567(0.1269) Grad: 4.0871  LR: 0.00000398  \n",
      "Epoch: [5][11000/38447] Elapsed 29m 29s (remain 73m 35s) Loss: 0.0090(0.1273) Grad: 0.3773  LR: 0.00000392  \n",
      "Epoch: [5][12000/38447] Elapsed 32m 9s (remain 70m 52s) Loss: 0.4463(0.1275) Grad: 10.2490  LR: 0.00000386  \n",
      "Epoch: [5][13000/38447] Elapsed 34m 50s (remain 68m 10s) Loss: 0.0178(0.1273) Grad: 1.3593  LR: 0.00000380  \n",
      "Epoch: [5][14000/38447] Elapsed 37m 30s (remain 65m 28s) Loss: 0.1043(0.1274) Grad: 8.1089  LR: 0.00000373  \n",
      "Epoch: [5][15000/38447] Elapsed 40m 10s (remain 62m 46s) Loss: 0.1787(0.1276) Grad: 6.3762  LR: 0.00000367  \n",
      "Epoch: [5][16000/38447] Elapsed 42m 50s (remain 60m 5s) Loss: 0.0773(0.1275) Grad: 1.8087  LR: 0.00000361  \n",
      "Epoch: [5][17000/38447] Elapsed 45m 30s (remain 57m 23s) Loss: 0.2005(0.1273) Grad: 10.9074  LR: 0.00000354  \n",
      "Epoch: [5][18000/38447] Elapsed 48m 10s (remain 54m 42s) Loss: 0.0794(0.1274) Grad: 6.3495  LR: 0.00000348  \n",
      "Epoch: [5][19000/38447] Elapsed 50m 50s (remain 52m 1s) Loss: 0.0442(0.1274) Grad: 2.2868  LR: 0.00000342  \n",
      "Epoch: [5][20000/38447] Elapsed 53m 30s (remain 49m 20s) Loss: 0.1068(0.1275) Grad: 3.4125  LR: 0.00000336  \n",
      "Epoch: [5][21000/38447] Elapsed 56m 10s (remain 46m 39s) Loss: 0.1590(0.1276) Grad: 14.7705  LR: 0.00000330  \n",
      "Epoch: [5][22000/38447] Elapsed 58m 50s (remain 43m 58s) Loss: 0.0216(0.1275) Grad: 1.9710  LR: 0.00000324  \n",
      "Epoch: [5][23000/38447] Elapsed 61m 30s (remain 41m 18s) Loss: 0.1531(0.1277) Grad: 6.1266  LR: 0.00000318  \n",
      "Epoch: [5][24000/38447] Elapsed 64m 15s (remain 38m 40s) Loss: 0.2282(0.1277) Grad: 5.2962  LR: 0.00000312  \n",
      "Epoch: [5][25000/38447] Elapsed 66m 59s (remain 36m 1s) Loss: 0.3831(0.1278) Grad: 13.5784  LR: 0.00000306  \n",
      "Epoch: [5][26000/38447] Elapsed 69m 41s (remain 33m 21s) Loss: 0.0302(0.1279) Grad: 2.9889  LR: 0.00000300  \n",
      "Epoch: [5][27000/38447] Elapsed 72m 25s (remain 30m 41s) Loss: 0.1477(0.1280) Grad: 10.9578  LR: 0.00000294  \n",
      "Epoch: [5][28000/38447] Elapsed 75m 16s (remain 28m 4s) Loss: 0.0390(0.1280) Grad: 2.7781  LR: 0.00000288  \n",
      "Epoch: [5][29000/38447] Elapsed 77m 59s (remain 25m 24s) Loss: 0.2872(0.1280) Grad: 16.2002  LR: 0.00000282  \n",
      "Epoch: [5][30000/38447] Elapsed 80m 9s (remain 22m 33s) Loss: 0.1785(0.1280) Grad: 20.0873  LR: 0.00000276  \n",
      "Epoch: [5][31000/38447] Elapsed 82m 19s (remain 19m 46s) Loss: 0.0744(0.1279) Grad: 3.2975  LR: 0.00000271  \n",
      "Epoch: [5][32000/38447] Elapsed 84m 24s (remain 17m 0s) Loss: 0.1712(0.1280) Grad: 37.5398  LR: 0.00000265  \n",
      "Epoch: [5][33000/38447] Elapsed 86m 29s (remain 14m 16s) Loss: 0.4460(0.1280) Grad: 16.9810  LR: 0.00000259  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [5][34000/38447] Elapsed 88m 36s (remain 11m 35s) Loss: 0.0844(0.1279) Grad: 5.0905  LR: 0.00000253  \n",
      "Epoch: [5][35000/38447] Elapsed 90m 59s (remain 8m 57s) Loss: 0.3017(0.1279) Grad: 13.7292  LR: 0.00000248  \n",
      "Epoch: [5][36000/38447] Elapsed 93m 14s (remain 6m 20s) Loss: 0.1122(0.1278) Grad: 4.6718  LR: 0.00000242  \n",
      "Epoch: [5][37000/38447] Elapsed 95m 18s (remain 3m 43s) Loss: 0.0136(0.1277) Grad: 0.7361  LR: 0.00000237  \n",
      "Epoch: [5][38000/38447] Elapsed 97m 38s (remain 1m 8s) Loss: 0.0185(0.1277) Grad: 1.2292  LR: 0.00000231  \n",
      "Epoch: [5][38446/38447] Elapsed 98m 36s (remain 0m 0s) Loss: 0.0171(0.1276) Grad: 2.0235  LR: 0.00000229  \n",
      "EVAL: [0/9613] Elapsed 0m 2s (remain 330m 31s) Loss: 2.5860(2.5860) \n",
      "EVAL: [1000/9613] Elapsed 0m 49s (remain 7m 5s) Loss: 0.2997(0.1786) \n",
      "EVAL: [2000/9613] Elapsed 1m 42s (remain 6m 30s) Loss: 0.2016(0.2212) \n",
      "EVAL: [3000/9613] Elapsed 2m 33s (remain 5m 38s) Loss: 0.0076(0.2298) \n",
      "EVAL: [4000/9613] Elapsed 3m 27s (remain 4m 51s) Loss: 0.1452(0.2446) \n",
      "EVAL: [5000/9613] Elapsed 4m 21s (remain 4m 1s) Loss: 0.0320(0.2555) \n",
      "EVAL: [6000/9613] Elapsed 5m 16s (remain 3m 10s) Loss: 0.4434(0.2643) \n",
      "EVAL: [7000/9613] Elapsed 6m 11s (remain 2m 18s) Loss: 0.5519(0.2755) \n",
      "EVAL: [8000/9613] Elapsed 7m 6s (remain 1m 25s) Loss: 0.3298(0.2917) \n",
      "EVAL: [9000/9613] Elapsed 8m 0s (remain 0m 32s) Loss: 0.0529(0.3104) \n",
      "EVAL: [9612/9613] Elapsed 8m 33s (remain 0m 0s) Loss: 1.1059(0.3302) \n",
      "Epoch 5 - avg_train_loss: 0.1276  avg_val_loss: 0.3302  time: 6694s\n",
      "Epoch 5 - Score: 0.5516 - Threshold: 0.02510\n",
      "Epoch 5 - Save: 0.5516 Model\n",
      "Epoch: [6][0/38447] Elapsed 0m 2s (remain 1393m 30s) Loss: 0.1572(0.1572) Grad: 10.0232  LR: 0.00000229  \n",
      "Epoch: [6][1000/38447] Elapsed 2m 5s (remain 78m 32s) Loss: 0.0455(0.0959) Grad: 10.3990  LR: 0.00000223  \n",
      "Epoch: [6][2000/38447] Elapsed 4m 14s (remain 77m 21s) Loss: 0.1718(0.0992) Grad: 8.4870  LR: 0.00000218  \n",
      "Epoch: [6][3000/38447] Elapsed 6m 31s (remain 77m 7s) Loss: 0.3291(0.0998) Grad: 20.1033  LR: 0.00000213  \n",
      "Epoch: [6][4000/38447] Elapsed 8m 37s (remain 74m 11s) Loss: 0.0459(0.1000) Grad: 0.8622  LR: 0.00000207  \n",
      "Epoch: [6][5000/38447] Elapsed 10m 41s (remain 71m 33s) Loss: 0.0096(0.1009) Grad: 0.6598  LR: 0.00000202  \n",
      "Epoch: [6][6000/38447] Elapsed 12m 46s (remain 69m 6s) Loss: 0.3200(0.1014) Grad: 5.4628  LR: 0.00000197  \n",
      "Epoch: [6][7000/38447] Elapsed 14m 54s (remain 66m 57s) Loss: 0.0671(0.1016) Grad: 5.4589  LR: 0.00000192  \n",
      "Epoch: [6][8000/38447] Elapsed 17m 3s (remain 64m 54s) Loss: 0.2437(0.1015) Grad: 9.1279  LR: 0.00000187  \n",
      "Epoch: [6][9000/38447] Elapsed 19m 13s (remain 62m 52s) Loss: 0.0238(0.1013) Grad: 2.4212  LR: 0.00000182  \n",
      "Epoch: [6][10000/38447] Elapsed 21m 45s (remain 61m 52s) Loss: 0.0450(0.1016) Grad: 6.6675  LR: 0.00000177  \n",
      "Epoch: [6][11000/38447] Elapsed 24m 25s (remain 60m 56s) Loss: 0.0067(0.1017) Grad: 0.6781  LR: 0.00000172  \n",
      "Epoch: [6][12000/38447] Elapsed 27m 0s (remain 59m 30s) Loss: 0.0128(0.1019) Grad: 3.6423  LR: 0.00000167  \n",
      "Epoch: [6][13000/38447] Elapsed 29m 34s (remain 57m 53s) Loss: 0.0366(0.1021) Grad: 7.8614  LR: 0.00000162  \n",
      "Epoch: [6][14000/38447] Elapsed 32m 3s (remain 55m 57s) Loss: 0.0068(0.1020) Grad: 0.1803  LR: 0.00000157  \n",
      "Epoch: [6][15000/38447] Elapsed 34m 25s (remain 53m 47s) Loss: 0.0196(0.1024) Grad: 2.1558  LR: 0.00000153  \n",
      "Epoch: [6][16000/38447] Elapsed 36m 43s (remain 51m 30s) Loss: 0.2003(0.1025) Grad: 5.3289  LR: 0.00000148  \n",
      "Epoch: [6][17000/38447] Elapsed 39m 10s (remain 49m 24s) Loss: 0.0189(0.1029) Grad: 1.7754  LR: 0.00000143  \n",
      "Epoch: [6][18000/38447] Elapsed 41m 34s (remain 47m 13s) Loss: 0.1171(0.1031) Grad: 10.7173  LR: 0.00000139  \n",
      "Epoch: [6][19000/38447] Elapsed 43m 52s (remain 44m 54s) Loss: 0.1714(0.1032) Grad: 14.1551  LR: 0.00000134  \n",
      "Epoch: [6][20000/38447] Elapsed 46m 4s (remain 42m 29s) Loss: 0.0588(0.1031) Grad: 5.0069  LR: 0.00000130  \n",
      "Epoch: [6][21000/38447] Elapsed 48m 14s (remain 40m 4s) Loss: 0.0566(0.1032) Grad: 1.3031  LR: 0.00000126  \n",
      "Epoch: [6][22000/38447] Elapsed 50m 26s (remain 37m 42s) Loss: 0.0620(0.1033) Grad: 3.7454  LR: 0.00000121  \n",
      "Epoch: [6][23000/38447] Elapsed 52m 44s (remain 35m 25s) Loss: 0.1140(0.1034) Grad: 16.8314  LR: 0.00000117  \n",
      "Epoch: [6][24000/38447] Elapsed 54m 55s (remain 33m 3s) Loss: 0.0325(0.1035) Grad: 6.3020  LR: 0.00000113  \n",
      "Epoch: [6][25000/38447] Elapsed 57m 9s (remain 30m 44s) Loss: 0.2370(0.1032) Grad: 49.8606  LR: 0.00000109  \n",
      "Epoch: [6][26000/38447] Elapsed 59m 22s (remain 28m 25s) Loss: 0.0730(0.1031) Grad: 1.6679  LR: 0.00000105  \n",
      "Epoch: [6][27000/38447] Elapsed 62m 1s (remain 26m 17s) Loss: 0.0527(0.1032) Grad: 1.5634  LR: 0.00000101  \n",
      "Epoch: [6][28000/38447] Elapsed 64m 43s (remain 24m 8s) Loss: 0.0058(0.1030) Grad: 0.8133  LR: 0.00000097  \n",
      "Epoch: [6][29000/38447] Elapsed 66m 55s (remain 21m 47s) Loss: 0.2402(0.1030) Grad: 8.8645  LR: 0.00000093  \n",
      "Epoch: [6][30000/38447] Elapsed 69m 11s (remain 19m 28s) Loss: 0.0972(0.1029) Grad: 3.0892  LR: 0.00000090  \n",
      "Epoch: [6][31000/38447] Elapsed 71m 19s (remain 17m 7s) Loss: 0.0069(0.1030) Grad: 1.1954  LR: 0.00000086  \n",
      "Epoch: [6][32000/38447] Elapsed 73m 28s (remain 14m 47s) Loss: 0.0775(0.1031) Grad: 12.6805  LR: 0.00000082  \n",
      "Epoch: [6][33000/38447] Elapsed 75m 33s (remain 12m 28s) Loss: 0.0125(0.1031) Grad: 0.4159  LR: 0.00000079  \n",
      "Epoch: [6][34000/38447] Elapsed 77m 55s (remain 10m 11s) Loss: 0.0075(0.1031) Grad: 2.7836  LR: 0.00000075  \n",
      "Epoch: [6][35000/38447] Elapsed 80m 0s (remain 7m 52s) Loss: 0.1209(0.1031) Grad: 13.0294  LR: 0.00000072  \n",
      "Epoch: [6][36000/38447] Elapsed 82m 5s (remain 5m 34s) Loss: 0.1587(0.1032) Grad: 7.9784  LR: 0.00000069  \n",
      "Epoch: [6][37000/38447] Elapsed 84m 10s (remain 3m 17s) Loss: 0.2663(0.1031) Grad: 17.9549  LR: 0.00000065  \n",
      "Epoch: [6][38000/38447] Elapsed 86m 14s (remain 1m 0s) Loss: 0.0065(0.1030) Grad: 2.4361  LR: 0.00000062  \n",
      "Epoch: [6][38446/38447] Elapsed 87m 10s (remain 0m 0s) Loss: 0.0795(0.1030) Grad: 7.6124  LR: 0.00000061  \n",
      "EVAL: [0/9613] Elapsed 0m 2s (remain 331m 21s) Loss: 3.5694(3.5694) \n",
      "EVAL: [1000/9613] Elapsed 0m 48s (remain 6m 57s) Loss: 0.3188(0.2188) \n",
      "EVAL: [2000/9613] Elapsed 1m 41s (remain 6m 27s) Loss: 0.2617(0.2694) \n",
      "EVAL: [3000/9613] Elapsed 2m 31s (remain 5m 34s) Loss: 0.0014(0.2800) \n",
      "EVAL: [4000/9613] Elapsed 3m 26s (remain 4m 49s) Loss: 0.1438(0.3004) \n",
      "EVAL: [5000/9613] Elapsed 4m 20s (remain 3m 59s) Loss: 0.0239(0.3149) \n",
      "EVAL: [6000/9613] Elapsed 5m 15s (remain 3m 9s) Loss: 0.5099(0.3258) \n",
      "EVAL: [7000/9613] Elapsed 6m 10s (remain 2m 18s) Loss: 0.6787(0.3406) \n",
      "EVAL: [8000/9613] Elapsed 7m 4s (remain 1m 25s) Loss: 0.4076(0.3617) \n",
      "EVAL: [9000/9613] Elapsed 7m 59s (remain 0m 32s) Loss: 0.0335(0.3854) \n",
      "EVAL: [9612/9613] Elapsed 8m 32s (remain 0m 0s) Loss: 1.4681(0.4100) \n",
      "Epoch 6 - avg_train_loss: 0.1030  avg_val_loss: 0.4100  time: 6018s\n",
      "Epoch 6 - Score: 0.5538 - Threshold: 0.00810\n",
      "Epoch 6 - Save: 0.5538 Model\n",
      "Epoch: [7][0/38447] Elapsed 0m 2s (remain 1822m 6s) Loss: 0.2984(0.2984) Grad: 15.3373  LR: 0.00000061  \n",
      "Epoch: [7][1000/38447] Elapsed 2m 41s (remain 100m 47s) Loss: 0.0094(0.0884) Grad: 0.8547  LR: 0.00000058  \n",
      "Epoch: [7][2000/38447] Elapsed 5m 24s (remain 98m 26s) Loss: 0.0026(0.0896) Grad: 0.2526  LR: 0.00000055  \n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_150609/785370676.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Train and evaluate one fold\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# Stage 1 Max Recall: 8468\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain_and_evaluate_one_fold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcorrelations\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCFG\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/tmp/ipykernel_150609/440096125.py\u001b[0m in \u001b[0;36mtrain_and_evaluate_one_fold\u001b[0;34m(train, correlations, fold, cfg)\u001b[0m\n\u001b[1;32m    402\u001b[0m         \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    403\u001b[0m         \u001b[0;31m# Train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 404\u001b[0;31m         \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    405\u001b[0m         \u001b[0;31m# Validation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    406\u001b[0m         \u001b[0mavg_val_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalid_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_150609/440096125.py\u001b[0m in \u001b[0;36mtrain_fn\u001b[0;34m(train_loader, model, criterion, optimizer, epoch, scheduler, device, cfg)\u001b[0m\n\u001b[1;32m    253\u001b[0m         \u001b[0mscaler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munscale_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m         \u001b[0mgrad_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_grad_norm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 255\u001b[0;31m         \u001b[0mscaler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    256\u001b[0m         \u001b[0mscaler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/torch/cuda/amp/grad_scaler.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, optimizer, *args, **kwargs)\u001b[0m\n\u001b[1;32m    339\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer_state\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"found_inf_per_device\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"No inf checks were recorded for this optimizer.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    340\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 341\u001b[0;31m         \u001b[0mretval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_opt_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    343\u001b[0m         \u001b[0moptimizer_state\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"stage\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOptState\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSTEPPED\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/torch/cuda/amp/grad_scaler.py\u001b[0m in \u001b[0;36m_maybe_opt_step\u001b[0;34m(self, optimizer, optimizer_state, *args, **kwargs)\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0mretval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0moptimizer_state\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"found_inf_per_device\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 288\u001b[0;31m             \u001b[0mretval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    289\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mretval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    290\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/torch/optim/lr_scheduler.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     66\u001b[0m                 \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step_count\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m                 \u001b[0mwrapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m             \u001b[0;31m# Note that the returned function here is no longer a bound method,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    138\u001b[0m                 \u001b[0mprofile_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Optimizer.step#{}.step\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m                     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    141\u001b[0m                     \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_step_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/torch/optim/adamw.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    160\u001b[0m                 \u001b[0mstate_steps\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'step'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m             adamw(params_with_grad,\n\u001b[0m\u001b[1;32m    163\u001b[0m                   \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m                   \u001b[0mexp_avgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/torch/optim/adamw.py\u001b[0m in \u001b[0;36madamw\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, foreach, capturable, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize)\u001b[0m\n\u001b[1;32m    217\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_single_tensor_adamw\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 219\u001b[0;31m     func(params,\n\u001b[0m\u001b[1;32m    220\u001b[0m          \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m          \u001b[0mexp_avgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/torch/optim/adamw.py\u001b[0m in \u001b[0;36m_single_tensor_adamw\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize, capturable)\u001b[0m\n\u001b[1;32m    314\u001b[0m                 \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sqs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction2_sqrt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m                 \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction2_sqrt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m             \u001b[0mparam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcdiv_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexp_avg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdenom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstep_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Train and evaluate one fold\n",
    "# Stage 1 Max Recall: 8468\n",
    "train_and_evaluate_one_fold(train, correlations, 0, CFG)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "========== fold: 0 training ==========\n",
    "Epoch: [1][0/78269] Elapsed 0m 1s (remain 2260m 36s) Loss: 0.7120(0.7120) Grad: 1.5701  LR: 0.00000000  \n",
    "Epoch: [1][1000/78269] Elapsed 1m 26s (remain 111m 22s) Loss: 0.4438(0.6271) Grad: 1.5454  LR: 0.00000021  \n",
    "Epoch: [1][2000/78269] Elapsed 2m 51s (remain 109m 7s) Loss: 0.3089(0.4817) Grad: 0.3681  LR: 0.00000043  \n",
    "Epoch: [1][3000/78269] Elapsed 4m 17s (remain 107m 31s) Loss: 0.2367(0.4207) Grad: 0.5546  LR: 0.00000064  \n",
    "Epoch: [1][4000/78269] Elapsed 5m 40s (remain 105m 25s) Loss: 0.1598(0.3892) Grad: 0.8652  LR: 0.00000085  \n",
    "Epoch: [1][5000/78269] Elapsed 7m 3s (remain 103m 30s) Loss: 0.2463(0.3702) Grad: 0.9135  LR: 0.00000106  \n",
    "Epoch: [1][6000/78269] Elapsed 8m 27s (remain 101m 46s) Loss: 0.3828(0.3560) Grad: 1.8164  LR: 0.00000128  \n",
    "Epoch: [1][7000/78269] Elapsed 9m 50s (remain 100m 8s) Loss: 0.3850(0.3452) Grad: 3.1474  LR: 0.00000149  \n",
    "Epoch: [1][8000/78269] Elapsed 11m 13s (remain 98m 36s) Loss: 0.3472(0.3370) Grad: 1.9940  LR: 0.00000170  \n",
    "Epoch: [1][9000/78269] Elapsed 12m 36s (remain 97m 4s) Loss: 0.1438(0.3303) Grad: 1.3507  LR: 0.00000192  \n",
    "Epoch: [1][10000/78269] Elapsed 14m 0s (remain 95m 36s) Loss: 0.2023(0.3241) Grad: 2.2278  LR: 0.00000213  \n",
    "Epoch: [1][11000/78269] Elapsed 15m 23s (remain 94m 7s) Loss: 0.1871(0.3191) Grad: 2.0018  LR: 0.00000234  \n",
    "Epoch: [1][12000/78269] Elapsed 16m 46s (remain 92m 39s) Loss: 0.2495(0.3149) Grad: 2.9007  LR: 0.00000256  \n",
    "Epoch: [1][13000/78269] Elapsed 18m 10s (remain 91m 14s) Loss: 0.4534(0.3111) Grad: 3.5416  LR: 0.00000277  \n",
    "Epoch: [1][14000/78269] Elapsed 19m 33s (remain 89m 48s) Loss: 0.1983(0.3073) Grad: 2.3728  LR: 0.00000298  \n",
    "Epoch: [1][15000/78269] Elapsed 20m 56s (remain 88m 19s) Loss: 0.1458(0.3038) Grad: 1.3323  LR: 0.00000319  \n",
    "Epoch: [1][16000/78269] Elapsed 22m 18s (remain 86m 49s) Loss: 0.2105(0.3008) Grad: 2.2263  LR: 0.00000341  \n",
    "Epoch: [1][17000/78269] Elapsed 23m 41s (remain 85m 22s) Loss: 0.1893(0.2981) Grad: 3.1229  LR: 0.00000362  \n",
    "Epoch: [1][18000/78269] Elapsed 25m 3s (remain 83m 55s) Loss: 0.1889(0.2953) Grad: 2.9211  LR: 0.00000383  \n",
    "Epoch: [1][19000/78269] Elapsed 26m 26s (remain 82m 27s) Loss: 0.2973(0.2930) Grad: 4.3314  LR: 0.00000405  \n",
    "Epoch: [1][20000/78269] Elapsed 27m 48s (remain 81m 0s) Loss: 0.2057(0.2907) Grad: 1.7820  LR: 0.00000426  \n",
    "Epoch: [1][21000/78269] Elapsed 29m 10s (remain 79m 33s) Loss: 0.1743(0.2886) Grad: 2.4722  LR: 0.00000447  \n",
    "Epoch: [1][22000/78269] Elapsed 30m 32s (remain 78m 6s) Loss: 0.1655(0.2866) Grad: 4.4318  LR: 0.00000468  \n",
    "Epoch: [1][23000/78269] Elapsed 31m 54s (remain 76m 40s) Loss: 0.2084(0.2848) Grad: 2.5483  LR: 0.00000490  \n",
    "Epoch: [1][24000/78269] Elapsed 33m 16s (remain 75m 15s) Loss: 0.2242(0.2831) Grad: 1.7516  LR: 0.00000511  \n",
    "Epoch: [1][25000/78269] Elapsed 34m 38s (remain 73m 48s) Loss: 0.0640(0.2816) Grad: 1.4669  LR: 0.00000532  \n",
    "Epoch: [1][26000/78269] Elapsed 36m 0s (remain 72m 23s) Loss: 0.3006(0.2797) Grad: 3.1642  LR: 0.00000554  \n",
    "Epoch: [1][27000/78269] Elapsed 37m 22s (remain 70m 57s) Loss: 0.0847(0.2779) Grad: 4.5314  LR: 0.00000575  \n",
    "Epoch: [1][28000/78269] Elapsed 38m 44s (remain 69m 33s) Loss: 0.2188(0.2764) Grad: 2.8374  LR: 0.00000596  \n",
    "Epoch: [1][29000/78269] Elapsed 40m 6s (remain 68m 8s) Loss: 0.3947(0.2747) Grad: 2.6061  LR: 0.00000618  \n",
    "Epoch: [1][30000/78269] Elapsed 41m 29s (remain 66m 45s) Loss: 0.2308(0.2735) Grad: 2.4709  LR: 0.00000639  \n",
    "Epoch: [1][31000/78269] Elapsed 42m 51s (remain 65m 21s) Loss: 0.2755(0.2723) Grad: 1.8329  LR: 0.00000660  \n",
    "Epoch: [1][32000/78269] Elapsed 44m 14s (remain 63m 57s) Loss: 0.2371(0.2709) Grad: 2.4145  LR: 0.00000681  \n",
    "Epoch: [1][33000/78269] Elapsed 45m 36s (remain 62m 34s) Loss: 0.1392(0.2697) Grad: 1.5320  LR: 0.00000703  \n",
    "Epoch: [1][34000/78269] Elapsed 46m 59s (remain 61m 10s) Loss: 0.1857(0.2685) Grad: 2.9083  LR: 0.00000724  \n",
    "Epoch: [1][35000/78269] Elapsed 48m 21s (remain 59m 46s) Loss: 0.2306(0.2672) Grad: 2.1018  LR: 0.00000745  \n",
    "Epoch: [1][36000/78269] Elapsed 49m 43s (remain 58m 23s) Loss: 0.1146(0.2660) Grad: 1.7258  LR: 0.00000767  \n",
    "Epoch: [1][37000/78269] Elapsed 51m 6s (remain 56m 59s) Loss: 0.2881(0.2650) Grad: 1.9609  LR: 0.00000788  \n",
    "Epoch: [1][38000/78269] Elapsed 52m 28s (remain 55m 36s) Loss: 0.1950(0.2640) Grad: 2.0378  LR: 0.00000809  \n",
    "Epoch: [1][39000/78269] Elapsed 53m 51s (remain 54m 13s) Loss: 0.3010(0.2630) Grad: 4.4506  LR: 0.00000830  \n",
    "Epoch: [1][40000/78269] Elapsed 55m 13s (remain 52m 49s) Loss: 0.3181(0.2621) Grad: 2.8970  LR: 0.00000852  \n",
    "Epoch: [1][41000/78269] Elapsed 56m 35s (remain 51m 26s) Loss: 0.3358(0.2611) Grad: 2.8687  LR: 0.00000873  \n",
    "Epoch: [1][42000/78269] Elapsed 57m 57s (remain 50m 3s) Loss: 0.3176(0.2602) Grad: 2.0962  LR: 0.00000894  \n",
    "Epoch: [1][43000/78269] Elapsed 59m 20s (remain 48m 39s) Loss: 0.0568(0.2592) Grad: 0.9268  LR: 0.00000916  \n",
    "Epoch: [1][44000/78269] Elapsed 60m 42s (remain 47m 16s) Loss: 0.0637(0.2582) Grad: 2.8651  LR: 0.00000937  \n",
    "Epoch: [1][45000/78269] Elapsed 62m 4s (remain 45m 53s) Loss: 0.2726(0.2573) Grad: 5.7472  LR: 0.00000958  \n",
    "Epoch: [1][46000/78269] Elapsed 63m 26s (remain 44m 30s) Loss: 0.3803(0.2564) Grad: 2.5286  LR: 0.00000980  \n",
    "Epoch: [1][47000/78269] Elapsed 64m 49s (remain 43m 7s) Loss: 0.0446(0.2554) Grad: 0.8770  LR: 0.00001000  \n",
    "Epoch: [1][48000/78269] Elapsed 66m 11s (remain 41m 43s) Loss: 0.2244(0.2546) Grad: 3.8869  LR: 0.00001000  \n",
    "Epoch: [1][49000/78269] Elapsed 67m 33s (remain 40m 21s) Loss: 0.1609(0.2539) Grad: 2.1088  LR: 0.00001000  \n",
    "Epoch: [1][50000/78269] Elapsed 68m 55s (remain 38m 57s) Loss: 0.2398(0.2531) Grad: 2.0056  LR: 0.00001000  \n",
    "Epoch: [1][51000/78269] Elapsed 70m 17s (remain 37m 34s) Loss: 0.2350(0.2523) Grad: 2.8891  LR: 0.00001000  \n",
    "Epoch: [1][52000/78269] Elapsed 71m 39s (remain 36m 11s) Loss: 0.2421(0.2516) Grad: 2.7053  LR: 0.00001000  \n",
    "Epoch: [1][53000/78269] Elapsed 73m 1s (remain 34m 48s) Loss: 0.1218(0.2508) Grad: 1.6472  LR: 0.00000999  \n",
    "Epoch: [1][54000/78269] Elapsed 74m 23s (remain 33m 26s) Loss: 0.2621(0.2500) Grad: 6.2667  LR: 0.00000999  \n",
    "Epoch: [1][55000/78269] Elapsed 75m 45s (remain 32m 3s) Loss: 0.2797(0.2493) Grad: 1.7595  LR: 0.00000999  \n",
    "Epoch: [1][56000/78269] Elapsed 77m 8s (remain 30m 40s) Loss: 0.0275(0.2486) Grad: 0.8356  LR: 0.00000999  \n",
    "Epoch: [1][57000/78269] Elapsed 78m 30s (remain 29m 17s) Loss: 0.4394(0.2478) Grad: 4.1900  LR: 0.00000999  \n",
    "Epoch: [1][58000/78269] Elapsed 79m 52s (remain 27m 54s) Loss: 0.0763(0.2471) Grad: 1.3862  LR: 0.00000998  \n",
    "Epoch: [1][59000/78269] Elapsed 81m 14s (remain 26m 31s) Loss: 0.1655(0.2465) Grad: 1.4423  LR: 0.00000998  \n",
    "Epoch: [1][60000/78269] Elapsed 82m 37s (remain 25m 9s) Loss: 0.1615(0.2458) Grad: 1.8937  LR: 0.00000998  \n",
    "Epoch: [1][61000/78269] Elapsed 83m 59s (remain 23m 46s) Loss: 0.1440(0.2452) Grad: 1.6254  LR: 0.00000997  \n",
    "Epoch: [1][62000/78269] Elapsed 85m 21s (remain 22m 23s) Loss: 0.1134(0.2446) Grad: 2.3672  LR: 0.00000997  \n",
    "Epoch: [1][63000/78269] Elapsed 86m 44s (remain 21m 1s) Loss: 0.2467(0.2439) Grad: 3.7494  LR: 0.00000996  \n",
    "Epoch: [1][64000/78269] Elapsed 88m 6s (remain 19m 38s) Loss: 0.0889(0.2433) Grad: 1.0698  LR: 0.00000996  \n",
    "Epoch: [1][65000/78269] Elapsed 89m 28s (remain 18m 15s) Loss: 0.1732(0.2427) Grad: 2.4345  LR: 0.00000996  \n",
    "Epoch: [1][66000/78269] Elapsed 90m 50s (remain 16m 53s) Loss: 0.0685(0.2421) Grad: 1.1233  LR: 0.00000995  \n",
    "Epoch: [1][67000/78269] Elapsed 92m 13s (remain 15m 30s) Loss: 0.1262(0.2415) Grad: 2.1719  LR: 0.00000994  \n",
    "Epoch: [1][68000/78269] Elapsed 93m 35s (remain 14m 7s) Loss: 0.1344(0.2409) Grad: 1.6418  LR: 0.00000994  \n",
    "Epoch: [1][69000/78269] Elapsed 94m 57s (remain 12m 45s) Loss: 0.0877(0.2404) Grad: 0.9137  LR: 0.00000993  \n",
    "Epoch: [1][70000/78269] Elapsed 96m 20s (remain 11m 22s) Loss: 0.1290(0.2399) Grad: 1.1786  LR: 0.00000993  \n",
    "Epoch: [1][71000/78269] Elapsed 97m 42s (remain 10m 0s) Loss: 0.3026(0.2395) Grad: 2.4442  LR: 0.00000992  \n",
    "Epoch: [1][72000/78269] Elapsed 99m 4s (remain 8m 37s) Loss: 0.2661(0.2389) Grad: 10.2384  LR: 0.00000991  \n",
    "Epoch: [1][73000/78269] Elapsed 100m 26s (remain 7m 14s) Loss: 0.0842(0.2383) Grad: 2.1289  LR: 0.00000991  \n",
    "Epoch: [1][74000/78269] Elapsed 101m 49s (remain 5m 52s) Loss: 0.0691(0.2378) Grad: 17.0663  LR: 0.00000990  \n",
    "Epoch: [1][75000/78269] Elapsed 103m 11s (remain 4m 29s) Loss: 0.2485(0.2373) Grad: 2.9377  LR: 0.00000989  \n",
    "Epoch: [1][76000/78269] Elapsed 104m 33s (remain 3m 7s) Loss: 0.3029(0.2368) Grad: 3.2337  LR: 0.00000988  \n",
    "Epoch: [1][77000/78269] Elapsed 105m 56s (remain 1m 44s) Loss: 0.2348(0.2362) Grad: 1.7777  LR: 0.00000988  \n",
    "Epoch: [1][78000/78269] Elapsed 107m 18s (remain 0m 22s) Loss: 0.2132(0.2357) Grad: 3.7708  LR: 0.00000987  \n",
    "Epoch: [1][78268/78269] Elapsed 107m 40s (remain 0m 0s) Loss: 0.2977(0.2355) Grad: 3.1356  LR: 0.00000987  \n",
    "EVAL: [0/19225] Elapsed 0m 0s (remain 320m 4s) Loss: 0.0162(0.0162) \n",
    "EVAL: [1000/19225] Elapsed 0m 13s (remain 4m 0s) Loss: 0.4839(0.1657) \n",
    "EVAL: [2000/19225] Elapsed 0m 24s (remain 3m 33s) Loss: 0.0008(0.1470) \n",
    "EVAL: [3000/19225] Elapsed 0m 35s (remain 3m 10s) Loss: 0.0008(0.1233) \n",
    "EVAL: [4000/19225] Elapsed 0m 48s (remain 3m 3s) Loss: 0.1303(0.1200) \n",
    "EVAL: [5000/19225] Elapsed 1m 1s (remain 2m 55s) Loss: 0.0074(0.1293) \n",
    "EVAL: [6000/19225] Elapsed 1m 15s (remain 2m 46s) Loss: 0.1010(0.1365) \n",
    "EVAL: [7000/19225] Elapsed 1m 29s (remain 2m 36s) Loss: 0.1494(0.1442) \n",
    "EVAL: [8000/19225] Elapsed 1m 44s (remain 2m 26s) Loss: 0.4008(0.1490) \n",
    "EVAL: [9000/19225] Elapsed 1m 59s (remain 2m 15s) Loss: 0.3023(0.1515) \n",
    "EVAL: [10000/19225] Elapsed 2m 14s (remain 2m 3s) Loss: 0.0901(0.1550) \n",
    "EVAL: [11000/19225] Elapsed 2m 29s (remain 1m 51s) Loss: 0.2495(0.1564) \n",
    "EVAL: [12000/19225] Elapsed 2m 45s (remain 1m 39s) Loss: 0.0063(0.1591) \n",
    "EVAL: [13000/19225] Elapsed 3m 0s (remain 1m 26s) Loss: 0.0733(0.1602) \n",
    "EVAL: [14000/19225] Elapsed 3m 17s (remain 1m 13s) Loss: 0.0047(0.1612) \n",
    "EVAL: [15000/19225] Elapsed 3m 34s (remain 1m 0s) Loss: 0.3380(0.1639) \n",
    "EVAL: [16000/19225] Elapsed 3m 51s (remain 0m 46s) Loss: 0.3581(0.1646) \n",
    "EVAL: [17000/19225] Elapsed 4m 9s (remain 0m 32s) Loss: 0.1334(0.1652) \n",
    "EVAL: [18000/19225] Elapsed 4m 28s (remain 0m 18s) Loss: 0.1469(0.1658) \n",
    "EVAL: [19000/19225] Elapsed 4m 49s (remain 0m 3s) Loss: 0.5381(0.1656) \n",
    "EVAL: [19224/19225] Elapsed 4m 54s (remain 0m 0s) Loss: 0.1550(0.1653) \n",
    "Epoch 1 - avg_train_loss: 0.2355  avg_val_loss: 0.1653  time: 6951s\n",
    "Epoch 1 - Score: 0.4112 - Threshold: 0.03700\n",
    "Epoch 1 - Save Best Score: 0.4112 Model\n",
    "Epoch: [2][0/78269] Elapsed 0m 1s (remain 1554m 30s) Loss: 0.2025(0.2025) Grad: 2.8341  LR: 0.00000987  \n",
    "Epoch: [2][1000/78269] Elapsed 1m 23s (remain 107m 33s) Loss: 0.0399(0.1889) Grad: 0.6968  LR: 0.00000986  \n",
    "Epoch: [2][2000/78269] Elapsed 2m 46s (remain 105m 32s) Loss: 0.1259(0.1852) Grad: 1.9818  LR: 0.00000985  \n",
    "Epoch: [2][3000/78269] Elapsed 4m 8s (remain 104m 4s) Loss: 0.2472(0.1834) Grad: 2.4412  LR: 0.00000984  \n",
    "Epoch: [2][4000/78269] Elapsed 5m 31s (remain 102m 38s) Loss: 0.1045(0.1826) Grad: 3.4805  LR: 0.00000983  \n",
    "Epoch: [2][5000/78269] Elapsed 6m 54s (remain 101m 14s) Loss: 0.3046(0.1834) Grad: 3.6378  LR: 0.00000982  \n",
    "Epoch: [2][6000/78269] Elapsed 8m 17s (remain 99m 52s) Loss: 0.0773(0.1837) Grad: 5.5224  LR: 0.00000981  \n",
    "Epoch: [2][7000/78269] Elapsed 9m 40s (remain 98m 26s) Loss: 0.3321(0.1840) Grad: 6.0008  LR: 0.00000980  \n",
    "Epoch: [2][8000/78269] Elapsed 11m 2s (remain 96m 55s) Loss: 0.0260(0.1842) Grad: 0.6304  LR: 0.00000979  \n",
    "Epoch: [2][9000/78269] Elapsed 12m 23s (remain 95m 25s) Loss: 0.0956(0.1842) Grad: 3.2497  LR: 0.00000978  \n",
    "Epoch: [2][10000/78269] Elapsed 13m 45s (remain 93m 57s) Loss: 0.1409(0.1839) Grad: 1.9610  LR: 0.00000977  \n",
    "Epoch: [2][11000/78269] Elapsed 15m 8s (remain 92m 32s) Loss: 0.1797(0.1841) Grad: 2.7107  LR: 0.00000975  \n",
    "Epoch: [2][12000/78269] Elapsed 16m 30s (remain 91m 8s) Loss: 0.2780(0.1842) Grad: 4.6539  LR: 0.00000974  \n",
    "Epoch: [2][13000/78269] Elapsed 17m 52s (remain 89m 46s) Loss: 0.3391(0.1842) Grad: 6.2706  LR: 0.00000973  \n",
    "Epoch: [2][14000/78269] Elapsed 19m 14s (remain 88m 20s) Loss: 0.1622(0.1843) Grad: 4.7731  LR: 0.00000972  \n",
    "Epoch: [2][15000/78269] Elapsed 20m 37s (remain 86m 58s) Loss: 0.2451(0.1838) Grad: 2.6115  LR: 0.00000971  \n",
    "Epoch: [2][16000/78269] Elapsed 21m 59s (remain 85m 34s) Loss: 0.1700(0.1837) Grad: 3.1568  LR: 0.00000969  \n",
    "Epoch: [2][17000/78269] Elapsed 23m 21s (remain 84m 11s) Loss: 0.2333(0.1836) Grad: 2.5243  LR: 0.00000968  \n",
    "Epoch: [2][18000/78269] Elapsed 24m 43s (remain 82m 47s) Loss: 0.3095(0.1834) Grad: 14.3975  LR: 0.00000967  \n",
    "Epoch: [2][19000/78269] Elapsed 26m 5s (remain 81m 22s) Loss: 0.0790(0.1835) Grad: 2.0043  LR: 0.00000965  \n",
    "Epoch: [2][20000/78269] Elapsed 27m 27s (remain 79m 59s) Loss: 0.0739(0.1835) Grad: 3.6580  LR: 0.00000964  \n",
    "Epoch: [2][21000/78269] Elapsed 28m 50s (remain 78m 38s) Loss: 0.1280(0.1836) Grad: 3.3149  LR: 0.00000963  \n",
    "Epoch: [2][22000/78269] Elapsed 30m 13s (remain 77m 18s) Loss: 0.0497(0.1835) Grad: 5.3077  LR: 0.00000961  \n",
    "Epoch: [2][23000/78269] Elapsed 31m 36s (remain 75m 57s) Loss: 0.1505(0.1834) Grad: 5.2129  LR: 0.00000960  \n",
    "Epoch: [2][24000/78269] Elapsed 32m 59s (remain 74m 35s) Loss: 0.1382(0.1833) Grad: 2.2067  LR: 0.00000958  \n",
    "Epoch: [2][25000/78269] Elapsed 34m 21s (remain 73m 12s) Loss: 0.0783(0.1833) Grad: 1.7962  LR: 0.00000957  \n",
    "Epoch: [2][26000/78269] Elapsed 35m 43s (remain 71m 49s) Loss: 0.1580(0.1832) Grad: 2.3131  LR: 0.00000955  \n",
    "Epoch: [2][27000/78269] Elapsed 37m 6s (remain 70m 26s) Loss: 0.3288(0.1829) Grad: 2.7680  LR: 0.00000954  \n",
    "Epoch: [2][28000/78269] Elapsed 38m 28s (remain 69m 3s) Loss: 0.2772(0.1827) Grad: 5.3784  LR: 0.00000952  \n",
    "Epoch: [2][29000/78269] Elapsed 39m 50s (remain 67m 40s) Loss: 0.3262(0.1826) Grad: 18.1003  LR: 0.00000951  \n",
    "Epoch: [2][30000/78269] Elapsed 41m 12s (remain 66m 17s) Loss: 0.4018(0.1825) Grad: 3.4773  LR: 0.00000949  \n",
    "Epoch: [2][31000/78269] Elapsed 42m 34s (remain 64m 54s) Loss: 0.0829(0.1824) Grad: 2.0196  LR: 0.00000947  \n",
    "Epoch: [2][32000/78269] Elapsed 43m 56s (remain 63m 31s) Loss: 0.0404(0.1822) Grad: 0.9983  LR: 0.00000946  \n",
    "Epoch: [2][33000/78269] Elapsed 45m 18s (remain 62m 8s) Loss: 0.5250(0.1820) Grad: 7.0574  LR: 0.00000944  \n",
    "Epoch: [2][34000/78269] Elapsed 46m 39s (remain 60m 45s) Loss: 0.3483(0.1819) Grad: 5.2442  LR: 0.00000942  \n",
    "Epoch: [2][35000/78269] Elapsed 48m 1s (remain 59m 22s) Loss: 0.3625(0.1817) Grad: 4.1163  LR: 0.00000940  \n",
    "Epoch: [2][36000/78269] Elapsed 49m 23s (remain 57m 59s) Loss: 0.0594(0.1814) Grad: 1.6955  LR: 0.00000939  \n",
    "Epoch: [2][37000/78269] Elapsed 50m 46s (remain 56m 37s) Loss: 0.0263(0.1813) Grad: 0.6898  LR: 0.00000937  \n",
    "Epoch: [2][38000/78269] Elapsed 52m 7s (remain 55m 14s) Loss: 0.0213(0.1813) Grad: 0.4150  LR: 0.00000935  \n",
    "Epoch: [2][39000/78269] Elapsed 53m 30s (remain 53m 52s) Loss: 0.1078(0.1811) Grad: 2.1407  LR: 0.00000933  \n",
    "Epoch: [2][40000/78269] Elapsed 54m 52s (remain 52m 29s) Loss: 0.4046(0.1811) Grad: 4.7918  LR: 0.00000931  \n",
    "Epoch: [2][41000/78269] Elapsed 56m 14s (remain 51m 7s) Loss: 0.1465(0.1809) Grad: 2.5341  LR: 0.00000930  \n",
    "Epoch: [2][42000/78269] Elapsed 57m 36s (remain 49m 44s) Loss: 0.2297(0.1807) Grad: 4.7288  LR: 0.00000928  \n",
    "Epoch: [2][43000/78269] Elapsed 58m 58s (remain 48m 21s) Loss: 0.0546(0.1805) Grad: 2.0774  LR: 0.00000926  \n",
    "Epoch: [2][44000/78269] Elapsed 60m 20s (remain 46m 59s) Loss: 0.1377(0.1804) Grad: 1.9747  LR: 0.00000924  \n",
    "Epoch: [2][45000/78269] Elapsed 61m 41s (remain 45m 36s) Loss: 0.3536(0.1802) Grad: 5.7389  LR: 0.00000922  \n",
    "Epoch: [2][46000/78269] Elapsed 63m 3s (remain 44m 14s) Loss: 0.0540(0.1800) Grad: 1.6381  LR: 0.00000920  \n",
    "Epoch: [2][47000/78269] Elapsed 64m 26s (remain 42m 51s) Loss: 0.1714(0.1800) Grad: 2.0969  LR: 0.00000918  \n",
    "Epoch: [2][48000/78269] Elapsed 65m 48s (remain 41m 29s) Loss: 0.4451(0.1798) Grad: 4.2398  LR: 0.00000916  \n",
    "Epoch: [2][49000/78269] Elapsed 67m 10s (remain 40m 7s) Loss: 0.0345(0.1797) Grad: 1.3840  LR: 0.00000914  \n",
    "Epoch: [2][50000/78269] Elapsed 68m 33s (remain 38m 45s) Loss: 0.1123(0.1795) Grad: 2.7178  LR: 0.00000911  \n",
    "Epoch: [2][51000/78269] Elapsed 69m 55s (remain 37m 23s) Loss: 0.3410(0.1793) Grad: 9.0874  LR: 0.00000909  \n",
    "Epoch: [2][52000/78269] Elapsed 71m 18s (remain 36m 1s) Loss: 0.1586(0.1791) Grad: 5.6084  LR: 0.00000907  \n",
    "Epoch: [2][53000/78269] Elapsed 72m 40s (remain 34m 38s) Loss: 0.1436(0.1790) Grad: 3.2307  LR: 0.00000905  \n",
    "Epoch: [2][54000/78269] Elapsed 74m 2s (remain 33m 16s) Loss: 0.2368(0.1789) Grad: 3.4278  LR: 0.00000903  \n",
    "Epoch: [2][55000/78269] Elapsed 75m 24s (remain 31m 54s) Loss: 0.2059(0.1787) Grad: 2.6555  LR: 0.00000901  \n",
    "Epoch: [2][56000/78269] Elapsed 76m 46s (remain 30m 31s) Loss: 0.1267(0.1786) Grad: 3.6763  LR: 0.00000898  \n",
    "Epoch: [2][57000/78269] Elapsed 78m 7s (remain 29m 9s) Loss: 0.1438(0.1785) Grad: 1.0754  LR: 0.00000896  \n",
    "Epoch: [2][58000/78269] Elapsed 79m 30s (remain 27m 46s) Loss: 0.0725(0.1783) Grad: 4.5157  LR: 0.00000894  \n",
    "Epoch: [2][59000/78269] Elapsed 80m 52s (remain 26m 24s) Loss: 0.0162(0.1782) Grad: 0.4782  LR: 0.00000892  \n",
    "Epoch: [2][60000/78269] Elapsed 82m 13s (remain 25m 2s) Loss: 0.0462(0.1780) Grad: 3.1942  LR: 0.00000889  \n",
    "Epoch: [2][61000/78269] Elapsed 83m 35s (remain 23m 39s) Loss: 0.1121(0.1779) Grad: 1.9557  LR: 0.00000887  \n",
    "Epoch: [2][62000/78269] Elapsed 84m 58s (remain 22m 17s) Loss: 0.1868(0.1778) Grad: 2.5406  LR: 0.00000884  \n",
    "Epoch: [2][63000/78269] Elapsed 86m 21s (remain 20m 55s) Loss: 0.1567(0.1777) Grad: 3.8759  LR: 0.00000882  \n",
    "Epoch: [2][64000/78269] Elapsed 87m 43s (remain 19m 33s) Loss: 0.0542(0.1776) Grad: 1.4913  LR: 0.00000880  \n",
    "Epoch: [2][65000/78269] Elapsed 89m 5s (remain 18m 11s) Loss: 0.2154(0.1774) Grad: 3.4836  LR: 0.00000877  \n",
    "Epoch: [2][66000/78269] Elapsed 90m 28s (remain 16m 48s) Loss: 0.2375(0.1773) Grad: 4.1597  LR: 0.00000875  \n",
    "Epoch: [2][67000/78269] Elapsed 91m 50s (remain 15m 26s) Loss: 0.6167(0.1772) Grad: 6.2454  LR: 0.00000872  \n",
    "Epoch: [2][68000/78269] Elapsed 93m 12s (remain 14m 4s) Loss: 0.1605(0.1770) Grad: 9.4820  LR: 0.00000870  \n",
    "Epoch: [2][69000/78269] Elapsed 94m 34s (remain 12m 42s) Loss: 0.2384(0.1768) Grad: 3.9497  LR: 0.00000867  \n",
    "Epoch: [2][70000/78269] Elapsed 95m 57s (remain 11m 20s) Loss: 0.2052(0.1767) Grad: 3.0865  LR: 0.00000865  \n",
    "Epoch: [2][71000/78269] Elapsed 97m 19s (remain 9m 57s) Loss: 0.3914(0.1765) Grad: 4.6967  LR: 0.00000862  \n",
    "Epoch: [2][72000/78269] Elapsed 98m 41s (remain 8m 35s) Loss: 0.1057(0.1764) Grad: 4.7622  LR: 0.00000860  \n",
    "Epoch: [2][73000/78269] Elapsed 100m 3s (remain 7m 13s) Loss: 0.4083(0.1763) Grad: 4.2763  LR: 0.00000857  \n",
    "Epoch: [2][74000/78269] Elapsed 101m 25s (remain 5m 50s) Loss: 0.1306(0.1762) Grad: 4.3913  LR: 0.00000854  \n",
    "Epoch: [2][75000/78269] Elapsed 102m 47s (remain 4m 28s) Loss: 0.0482(0.1761) Grad: 0.7264  LR: 0.00000852  \n",
    "Epoch: [2][76000/78269] Elapsed 104m 9s (remain 3m 6s) Loss: 0.3148(0.1759) Grad: 3.7670  LR: 0.00000849  \n",
    "Epoch: [2][77000/78269] Elapsed 105m 31s (remain 1m 44s) Loss: 0.0344(0.1758) Grad: 0.7638  LR: 0.00000847  \n",
    "Epoch: [2][78000/78269] Elapsed 106m 53s (remain 0m 22s) Loss: 0.1278(0.1757) Grad: 2.5362  LR: 0.00000844  \n",
    "Epoch: [2][78268/78269] Elapsed 107m 15s (remain 0m 0s) Loss: 0.1044(0.1756) Grad: 5.1211  LR: 0.00000843  \n",
    "EVAL: [0/19225] Elapsed 0m 1s (remain 341m 43s) Loss: 0.0150(0.0150) \n",
    "EVAL: [1000/19225] Elapsed 0m 13s (remain 4m 1s) Loss: 0.6518(0.1646) \n",
    "EVAL: [2000/19225] Elapsed 0m 24s (remain 3m 33s) Loss: 0.0003(0.1452) \n",
    "EVAL: [3000/19225] Elapsed 0m 35s (remain 3m 9s) Loss: 0.0004(0.1195) \n",
    "EVAL: [4000/19225] Elapsed 0m 47s (remain 3m 2s) Loss: 0.1219(0.1145) \n",
    "EVAL: [5000/19225] Elapsed 1m 1s (remain 2m 54s) Loss: 0.0060(0.1232) \n",
    "EVAL: [6000/19225] Elapsed 1m 15s (remain 2m 46s) Loss: 0.1560(0.1291) \n",
    "EVAL: [7000/19225] Elapsed 1m 29s (remain 2m 36s) Loss: 0.1396(0.1345) \n",
    "EVAL: [8000/19225] Elapsed 1m 44s (remain 2m 26s) Loss: 0.0964(0.1391) \n",
    "EVAL: [9000/19225] Elapsed 1m 58s (remain 2m 15s) Loss: 0.0593(0.1414) \n",
    "EVAL: [10000/19225] Elapsed 2m 14s (remain 2m 3s) Loss: 0.0829(0.1440) \n",
    "EVAL: [11000/19225] Elapsed 2m 29s (remain 1m 51s) Loss: 0.2283(0.1452) \n",
    "EVAL: [12000/19225] Elapsed 2m 44s (remain 1m 39s) Loss: 0.0038(0.1478) \n",
    "EVAL: [13000/19225] Elapsed 3m 0s (remain 1m 26s) Loss: 0.0322(0.1489) \n",
    "EVAL: [14000/19225] Elapsed 3m 17s (remain 1m 13s) Loss: 0.0065(0.1491) \n",
    "EVAL: [15000/19225] Elapsed 3m 33s (remain 1m 0s) Loss: 0.0434(0.1509) \n",
    "EVAL: [16000/19225] Elapsed 3m 51s (remain 0m 46s) Loss: 0.4001(0.1515) \n",
    "EVAL: [17000/19225] Elapsed 4m 9s (remain 0m 32s) Loss: 0.1224(0.1517) \n",
    "EVAL: [18000/19225] Elapsed 4m 28s (remain 0m 18s) Loss: 0.1166(0.1518) \n",
    "EVAL: [19000/19225] Elapsed 4m 48s (remain 0m 3s) Loss: 0.6019(0.1513) \n",
    "EVAL: [19224/19225] Elapsed 4m 54s (remain 0m 0s) Loss: 0.1739(0.1509) \n",
    "Epoch 2 - avg_train_loss: 0.1756  avg_val_loss: 0.1509  time: 6960s\n",
    "Epoch 2 - Score: 0.4517 - Threshold: 0.05200\n",
    "Epoch 2 - Save Best Score: 0.4517 Model\n",
    "Epoch: [3][0/78269] Elapsed 0m 1s (remain 1625m 3s) Loss: 0.1774(0.1774) Grad: 2.5465  LR: 0.00000843  \n",
    "Epoch: [3][1000/78269] Elapsed 1m 23s (remain 107m 20s) Loss: 0.1112(0.1407) Grad: 5.9497  LR: 0.00000840  \n",
    "Epoch: [3][2000/78269] Elapsed 2m 45s (remain 105m 20s) Loss: 0.0589(0.1405) Grad: 3.0625  LR: 0.00000838  \n",
    "Epoch: [3][3000/78269] Elapsed 4m 8s (remain 103m 43s) Loss: 0.1126(0.1423) Grad: 3.2302  LR: 0.00000835  \n",
    "Epoch: [3][4000/78269] Elapsed 5m 30s (remain 102m 11s) Loss: 0.0997(0.1417) Grad: 3.2495  LR: 0.00000832  \n",
    "Epoch: [3][5000/78269] Elapsed 6m 52s (remain 100m 49s) Loss: 0.0168(0.1426) Grad: 1.0159  LR: 0.00000829  \n",
    "Epoch: [3][6000/78269] Elapsed 8m 15s (remain 99m 24s) Loss: 0.1213(0.1430) Grad: 5.8144  LR: 0.00000827  \n",
    "Epoch: [3][7000/78269] Elapsed 9m 37s (remain 97m 59s) Loss: 0.4095(0.1433) Grad: 12.1648  LR: 0.00000824  \n",
    "Epoch: [3][8000/78269] Elapsed 11m 0s (remain 96m 36s) Loss: 0.0633(0.1429) Grad: 5.3389  LR: 0.00000821  \n",
    "Epoch: [3][9000/78269] Elapsed 12m 22s (remain 95m 12s) Loss: 0.0913(0.1430) Grad: 4.2671  LR: 0.00000818  \n",
    "Epoch: [3][10000/78269] Elapsed 13m 44s (remain 93m 48s) Loss: 0.0412(0.1427) Grad: 1.8418  LR: 0.00000815  \n",
    "Epoch: [3][11000/78269] Elapsed 15m 6s (remain 92m 22s) Loss: 0.1077(0.1432) Grad: 4.1621  LR: 0.00000812  \n",
    "Epoch: [3][12000/78269] Elapsed 16m 28s (remain 90m 56s) Loss: 0.0904(0.1432) Grad: 10.8576  LR: 0.00000809  \n",
    "Epoch: [3][13000/78269] Elapsed 17m 50s (remain 89m 32s) Loss: 0.0078(0.1436) Grad: 0.3570  LR: 0.00000806  \n",
    "Epoch: [3][14000/78269] Elapsed 19m 12s (remain 88m 8s) Loss: 0.1793(0.1437) Grad: 2.4320  LR: 0.00000803  \n",
    "Epoch: [3][15000/78269] Elapsed 20m 34s (remain 86m 45s) Loss: 0.1398(0.1435) Grad: 4.8244  LR: 0.00000801  \n",
    "Epoch: [3][16000/78269] Elapsed 21m 56s (remain 85m 23s) Loss: 0.3166(0.1437) Grad: 4.7457  LR: 0.00000798  \n",
    "Epoch: [3][17000/78269] Elapsed 23m 18s (remain 84m 0s) Loss: 0.0182(0.1437) Grad: 0.7381  LR: 0.00000795  \n",
    "Epoch: [3][18000/78269] Elapsed 24m 41s (remain 82m 39s) Loss: 0.1103(0.1437) Grad: 1.4073  LR: 0.00000792  \n",
    "Epoch: [3][19000/78269] Elapsed 26m 3s (remain 81m 17s) Loss: 0.3994(0.1438) Grad: 7.5770  LR: 0.00000789  \n",
    "Epoch: [3][20000/78269] Elapsed 27m 26s (remain 79m 56s) Loss: 0.0807(0.1438) Grad: 4.4626  LR: 0.00000785  \n",
    "Epoch: [3][21000/78269] Elapsed 28m 48s (remain 78m 34s) Loss: 0.0739(0.1437) Grad: 1.9062  LR: 0.00000782  \n",
    "Epoch: [3][22000/78269] Elapsed 30m 11s (remain 77m 12s) Loss: 0.2840(0.1437) Grad: 7.1349  LR: 0.00000779  \n",
    "Epoch: [3][23000/78269] Elapsed 31m 33s (remain 75m 49s) Loss: 0.0153(0.1438) Grad: 0.4523  LR: 0.00000776  \n",
    "Epoch: [3][24000/78269] Elapsed 32m 55s (remain 74m 27s) Loss: 0.1814(0.1436) Grad: 5.3543  LR: 0.00000773  \n",
    "Epoch: [3][25000/78269] Elapsed 34m 18s (remain 73m 5s) Loss: 0.2057(0.1436) Grad: 8.1152  LR: 0.00000770  \n",
    "Epoch: [3][26000/78269] Elapsed 35m 40s (remain 71m 43s) Loss: 0.0827(0.1436) Grad: 2.0687  LR: 0.00000767  \n",
    "Epoch: [3][27000/78269] Elapsed 37m 2s (remain 70m 20s) Loss: 0.1990(0.1435) Grad: 3.6222  LR: 0.00000764  \n",
    "Epoch: [3][28000/78269] Elapsed 38m 25s (remain 68m 58s) Loss: 0.1271(0.1435) Grad: 4.2146  LR: 0.00000761  \n",
    "Epoch: [3][29000/78269] Elapsed 39m 47s (remain 67m 35s) Loss: 0.1412(0.1435) Grad: 6.3729  LR: 0.00000757  \n",
    "Epoch: [3][30000/78269] Elapsed 41m 8s (remain 66m 11s) Loss: 0.0185(0.1436) Grad: 0.9966  LR: 0.00000754  \n",
    "Epoch: [3][31000/78269] Elapsed 42m 30s (remain 64m 48s) Loss: 0.0469(0.1435) Grad: 1.8109  LR: 0.00000751  \n",
    "Epoch: [3][32000/78269] Elapsed 43m 52s (remain 63m 26s) Loss: 0.0462(0.1436) Grad: 6.2127  LR: 0.00000748  \n",
    "Epoch: [3][33000/78269] Elapsed 45m 14s (remain 62m 3s) Loss: 0.0377(0.1436) Grad: 1.8423  LR: 0.00000745  \n",
    "Epoch: [3][34000/78269] Elapsed 46m 36s (remain 60m 40s) Loss: 0.0306(0.1436) Grad: 2.2957  LR: 0.00000741  \n",
    "Epoch: [3][35000/78269] Elapsed 47m 58s (remain 59m 18s) Loss: 0.2056(0.1435) Grad: 2.9757  LR: 0.00000738  \n",
    "Epoch: [3][36000/78269] Elapsed 49m 19s (remain 57m 55s) Loss: 0.0800(0.1435) Grad: 3.9836  LR: 0.00000735  \n",
    "Epoch: [3][37000/78269] Elapsed 50m 41s (remain 56m 32s) Loss: 0.1108(0.1435) Grad: 4.2843  LR: 0.00000731  \n",
    "Epoch: [3][38000/78269] Elapsed 52m 4s (remain 55m 10s) Loss: 0.3289(0.1435) Grad: 6.6986  LR: 0.00000728  \n",
    "Epoch: [3][39000/78269] Elapsed 53m 26s (remain 53m 48s) Loss: 0.1239(0.1434) Grad: 2.7497  LR: 0.00000725  \n",
    "Epoch: [3][40000/78269] Elapsed 54m 48s (remain 52m 25s) Loss: 0.1813(0.1435) Grad: 7.8206  LR: 0.00000722  \n",
    "Epoch: [3][41000/78269] Elapsed 56m 10s (remain 51m 3s) Loss: 0.1504(0.1434) Grad: 5.5100  LR: 0.00000718  \n",
    "Epoch: [3][42000/78269] Elapsed 57m 32s (remain 49m 41s) Loss: 0.1242(0.1434) Grad: 3.9862  LR: 0.00000715  \n",
    "Epoch: [3][43000/78269] Elapsed 58m 54s (remain 48m 19s) Loss: 0.1607(0.1433) Grad: 2.1083  LR: 0.00000711  \n",
    "Epoch: [3][44000/78269] Elapsed 60m 16s (remain 46m 56s) Loss: 0.1883(0.1433) Grad: 5.8072  LR: 0.00000708  \n",
    "Epoch: [3][45000/78269] Elapsed 61m 39s (remain 45m 34s) Loss: 0.1410(0.1433) Grad: 3.5307  LR: 0.00000705  \n",
    "Epoch: [3][46000/78269] Elapsed 63m 1s (remain 44m 12s) Loss: 0.1314(0.1433) Grad: 6.4447  LR: 0.00000701  \n",
    "Epoch: [3][47000/78269] Elapsed 64m 23s (remain 42m 49s) Loss: 0.1883(0.1433) Grad: 5.3653  LR: 0.00000698  \n",
    "Epoch: [3][48000/78269] Elapsed 65m 44s (remain 41m 27s) Loss: 0.0491(0.1433) Grad: 2.0615  LR: 0.00000694  \n",
    "Epoch: [3][49000/78269] Elapsed 67m 6s (remain 40m 5s) Loss: 0.2297(0.1432) Grad: 6.6034  LR: 0.00000691  \n",
    "Epoch: [3][50000/78269] Elapsed 68m 28s (remain 38m 42s) Loss: 0.0887(0.1431) Grad: 2.1417  LR: 0.00000688  \n",
    "Epoch: [3][51000/78269] Elapsed 69m 50s (remain 37m 20s) Loss: 0.1391(0.1430) Grad: 2.1794  LR: 0.00000684  \n",
    "Epoch: [3][52000/78269] Elapsed 71m 13s (remain 35m 58s) Loss: 0.2919(0.1429) Grad: 4.4615  LR: 0.00000681  \n",
    "Epoch: [3][53000/78269] Elapsed 72m 35s (remain 34m 36s) Loss: 0.0503(0.1429) Grad: 1.3734  LR: 0.00000677  \n",
    "Epoch: [3][54000/78269] Elapsed 73m 58s (remain 33m 14s) Loss: 0.2209(0.1429) Grad: 3.6470  LR: 0.00000674  \n",
    "Epoch: [3][55000/78269] Elapsed 75m 20s (remain 31m 52s) Loss: 0.1589(0.1429) Grad: 8.8402  LR: 0.00000670  \n",
    "Epoch: [3][56000/78269] Elapsed 76m 42s (remain 30m 30s) Loss: 0.0953(0.1429) Grad: 2.6187  LR: 0.00000667  \n",
    "Epoch: [3][57000/78269] Elapsed 78m 4s (remain 29m 8s) Loss: 0.0307(0.1429) Grad: 1.9651  LR: 0.00000663  \n",
    "Epoch: [3][58000/78269] Elapsed 79m 27s (remain 27m 46s) Loss: 0.1187(0.1428) Grad: 5.2935  LR: 0.00000660  \n",
    "Epoch: [3][59000/78269] Elapsed 80m 49s (remain 26m 23s) Loss: 0.1175(0.1428) Grad: 1.9497  LR: 0.00000656  \n",
    "Epoch: [3][60000/78269] Elapsed 82m 12s (remain 25m 1s) Loss: 0.0338(0.1427) Grad: 0.7761  LR: 0.00000653  \n",
    "Epoch: [3][61000/78269] Elapsed 83m 35s (remain 23m 39s) Loss: 0.1593(0.1427) Grad: 6.4811  LR: 0.00000649  \n",
    "Epoch: [3][62000/78269] Elapsed 84m 56s (remain 22m 17s) Loss: 0.1608(0.1427) Grad: 2.0383  LR: 0.00000646  \n",
    "Epoch: [3][63000/78269] Elapsed 86m 19s (remain 20m 55s) Loss: 0.1540(0.1426) Grad: 3.7394  LR: 0.00000642  \n",
    "Epoch: [3][64000/78269] Elapsed 87m 41s (remain 19m 32s) Loss: 0.3065(0.1426) Grad: 4.2110  LR: 0.00000638  \n",
    "Epoch: [3][65000/78269] Elapsed 89m 3s (remain 18m 10s) Loss: 0.2382(0.1425) Grad: 4.8881  LR: 0.00000635  \n",
    "Epoch: [3][66000/78269] Elapsed 90m 25s (remain 16m 48s) Loss: 0.3179(0.1425) Grad: 6.2530  LR: 0.00000631  \n",
    "Epoch: [3][67000/78269] Elapsed 91m 47s (remain 15m 26s) Loss: 0.4612(0.1425) Grad: 15.5437  LR: 0.00000628  \n",
    "Epoch: [3][68000/78269] Elapsed 93m 9s (remain 14m 3s) Loss: 0.0113(0.1425) Grad: 0.3660  LR: 0.00000624  \n",
    "Epoch: [3][69000/78269] Elapsed 94m 31s (remain 12m 41s) Loss: 0.0064(0.1424) Grad: 0.3354  LR: 0.00000621  \n",
    "Epoch: [3][70000/78269] Elapsed 95m 53s (remain 11m 19s) Loss: 0.0434(0.1423) Grad: 1.6838  LR: 0.00000617  \n",
    "Epoch: [3][71000/78269] Elapsed 97m 16s (remain 9m 57s) Loss: 0.0666(0.1422) Grad: 2.0335  LR: 0.00000613  \n",
    "Epoch: [3][72000/78269] Elapsed 98m 38s (remain 8m 35s) Loss: 0.0803(0.1422) Grad: 2.4764  LR: 0.00000610  \n",
    "Epoch: [3][73000/78269] Elapsed 100m 1s (remain 7m 13s) Loss: 0.1608(0.1422) Grad: 5.9266  LR: 0.00000606  \n",
    "Epoch: [3][74000/78269] Elapsed 101m 25s (remain 5m 50s) Loss: 0.0131(0.1421) Grad: 0.8371  LR: 0.00000602  \n",
    "Epoch: [3][75000/78269] Elapsed 102m 49s (remain 4m 28s) Loss: 0.1045(0.1420) Grad: 6.6890  LR: 0.00000599  \n",
    "Epoch: [3][76000/78269] Elapsed 104m 13s (remain 3m 6s) Loss: 0.0382(0.1420) Grad: 1.5747  LR: 0.00000595  \n",
    "Epoch: [3][77000/78269] Elapsed 105m 36s (remain 1m 44s) Loss: 0.1493(0.1419) Grad: 9.0514  LR: 0.00000591  \n",
    "Epoch: [3][78000/78269] Elapsed 107m 0s (remain 0m 22s) Loss: 0.1486(0.1419) Grad: 3.7138  LR: 0.00000588  \n",
    "Epoch: [3][78268/78269] Elapsed 107m 22s (remain 0m 0s) Loss: 0.0921(0.1419) Grad: 7.9862  LR: 0.00000587  \n",
    "EVAL: [0/19225] Elapsed 0m 1s (remain 339m 58s) Loss: 0.0150(0.0150) \n",
    "EVAL: [1000/19225] Elapsed 0m 13s (remain 4m 4s) Loss: 0.5902(0.1780) \n",
    "EVAL: [2000/19225] Elapsed 0m 25s (remain 3m 36s) Loss: 0.0001(0.1543) \n",
    "EVAL: [3000/19225] Elapsed 0m 35s (remain 3m 12s) Loss: 0.0002(0.1248) \n",
    "EVAL: [4000/19225] Elapsed 0m 48s (remain 3m 5s) Loss: 0.0901(0.1186) \n",
    "EVAL: [5000/19225] Elapsed 1m 2s (remain 2m 58s) Loss: 0.0130(0.1290) \n",
    "EVAL: [6000/19225] Elapsed 1m 16s (remain 2m 49s) Loss: 0.2311(0.1349) \n",
    "EVAL: [7000/19225] Elapsed 1m 31s (remain 2m 39s) Loss: 0.0826(0.1389) \n",
    "EVAL: [8000/19225] Elapsed 1m 45s (remain 2m 28s) Loss: 0.0312(0.1433) \n",
    "EVAL: [9000/19225] Elapsed 2m 0s (remain 2m 17s) Loss: 0.0056(0.1481) \n",
    "EVAL: [10000/19225] Elapsed 2m 16s (remain 2m 5s) Loss: 0.0348(0.1504) \n",
    "EVAL: [11000/19225] Elapsed 2m 31s (remain 1m 53s) Loss: 0.1647(0.1510) \n",
    "EVAL: [12000/19225] Elapsed 2m 47s (remain 1m 41s) Loss: 0.0009(0.1534) \n",
    "EVAL: [13000/19225] Elapsed 3m 4s (remain 1m 28s) Loss: 0.0093(0.1540) \n",
    "EVAL: [14000/19225] Elapsed 3m 20s (remain 1m 14s) Loss: 0.0020(0.1535) \n",
    "EVAL: [15000/19225] Elapsed 3m 37s (remain 1m 1s) Loss: 0.0103(0.1548) \n",
    "EVAL: [16000/19225] Elapsed 3m 54s (remain 0m 47s) Loss: 0.4392(0.1554) \n",
    "EVAL: [17000/19225] Elapsed 4m 13s (remain 0m 33s) Loss: 0.0658(0.1553) \n",
    "EVAL: [18000/19225] Elapsed 4m 32s (remain 0m 18s) Loss: 0.0358(0.1549) \n",
    "EVAL: [19000/19225] Elapsed 4m 53s (remain 0m 3s) Loss: 0.6194(0.1542) \n",
    "EVAL: [19224/19225] Elapsed 4m 58s (remain 0m 0s) Loss: 0.1502(0.1540) \n",
    "Epoch 3 - avg_train_loss: 0.1419  avg_val_loss: 0.1540  time: 6990s\n",
    "Epoch 3 - Score: 0.4759 - Threshold: 0.05200\n",
    "Epoch 3 - Save Best Score: 0.4759 Model\n",
    "Epoch: [4][0/78269] Elapsed 0m 1s (remain 1672m 38s) Loss: 0.0836(0.0836) Grad: 3.6663  LR: 0.00000587  \n",
    "Epoch: [4][1000/78269] Elapsed 1m 24s (remain 109m 12s) Loss: 0.3014(0.1149) Grad: 7.9873  LR: 0.00000583  \n",
    "Epoch: [4][2000/78269] Elapsed 2m 48s (remain 107m 10s) Loss: 0.1561(0.1137) Grad: 12.8383  LR: 0.00000579  \n",
    "Epoch: [4][3000/78269] Elapsed 4m 12s (remain 105m 28s) Loss: 0.1692(0.1139) Grad: 4.0225  LR: 0.00000576  \n",
    "Epoch: [4][4000/78269] Elapsed 5m 35s (remain 103m 54s) Loss: 0.0170(0.1138) Grad: 0.8162  LR: 0.00000572  \n",
    "Epoch: [4][5000/78269] Elapsed 6m 59s (remain 102m 19s) Loss: 0.1645(0.1147) Grad: 4.8377  LR: 0.00000568  \n",
    "Epoch: [4][6000/78269] Elapsed 8m 22s (remain 100m 50s) Loss: 0.1896(0.1148) Grad: 3.9425  LR: 0.00000565  \n",
    "Epoch: [4][7000/78269] Elapsed 9m 45s (remain 99m 21s) Loss: 0.3016(0.1146) Grad: 10.8695  LR: 0.00000561  \n",
    "Epoch: [4][8000/78269] Elapsed 11m 9s (remain 98m 3s) Loss: 0.0034(0.1148) Grad: 0.2040  LR: 0.00000557  \n",
    "Epoch: [4][9000/78269] Elapsed 12m 34s (remain 96m 43s) Loss: 0.0238(0.1150) Grad: 1.5024  LR: 0.00000554  \n",
    "Epoch: [4][10000/78269] Elapsed 13m 57s (remain 95m 15s) Loss: 0.2139(0.1156) Grad: 7.1529  LR: 0.00000550  \n",
    "Epoch: [4][11000/78269] Elapsed 15m 20s (remain 93m 49s) Loss: 0.1891(0.1154) Grad: 11.3115  LR: 0.00000546  \n",
    "Epoch: [4][12000/78269] Elapsed 16m 43s (remain 92m 22s) Loss: 0.2439(0.1156) Grad: 8.2289  LR: 0.00000543  \n",
    "Epoch: [4][13000/78269] Elapsed 18m 7s (remain 91m 0s) Loss: 0.0541(0.1157) Grad: 1.8750  LR: 0.00000539  \n",
    "Epoch: [4][14000/78269] Elapsed 19m 30s (remain 89m 31s) Loss: 0.1161(0.1157) Grad: 4.3886  LR: 0.00000535  \n",
    "Epoch: [4][15000/78269] Elapsed 20m 52s (remain 88m 1s) Loss: 0.0390(0.1155) Grad: 3.1928  LR: 0.00000532  \n",
    "Epoch: [4][16000/78269] Elapsed 22m 14s (remain 86m 32s) Loss: 0.0378(0.1155) Grad: 2.6146  LR: 0.00000528  \n",
    "Epoch: [4][17000/78269] Elapsed 23m 36s (remain 85m 3s) Loss: 0.1264(0.1153) Grad: 9.4748  LR: 0.00000524  \n",
    "Epoch: [4][18000/78269] Elapsed 24m 57s (remain 83m 35s) Loss: 0.0534(0.1153) Grad: 2.4955  LR: 0.00000520  \n",
    "Epoch: [4][19000/78269] Elapsed 26m 21s (remain 82m 11s) Loss: 0.0089(0.1154) Grad: 0.6422  LR: 0.00000517  \n",
    "Epoch: [4][20000/78269] Elapsed 27m 43s (remain 80m 45s) Loss: 0.1634(0.1156) Grad: 5.2496  LR: 0.00000513  \n",
    "Epoch: [4][21000/78269] Elapsed 29m 5s (remain 79m 19s) Loss: 0.1241(0.1160) Grad: 2.0833  LR: 0.00000509  \n",
    "Epoch: [4][22000/78269] Elapsed 30m 27s (remain 77m 52s) Loss: 0.0983(0.1160) Grad: 4.1797  LR: 0.00000506  \n",
    "Epoch: [4][23000/78269] Elapsed 31m 48s (remain 76m 26s) Loss: 0.2882(0.1163) Grad: 7.0536  LR: 0.00000502  \n",
    "Epoch: [4][24000/78269] Elapsed 33m 10s (remain 75m 1s) Loss: 0.1114(0.1163) Grad: 9.1775  LR: 0.00000498  \n",
    "Epoch: [4][25000/78269] Elapsed 34m 32s (remain 73m 36s) Loss: 0.0625(0.1164) Grad: 5.5776  LR: 0.00000494  \n",
    "Epoch: [4][26000/78269] Elapsed 35m 54s (remain 72m 11s) Loss: 0.0213(0.1163) Grad: 3.0613  LR: 0.00000491  \n",
    "Epoch: [4][27000/78269] Elapsed 37m 18s (remain 70m 49s) Loss: 0.0242(0.1163) Grad: 1.1685  LR: 0.00000487  \n",
    "Epoch: [4][28000/78269] Elapsed 38m 40s (remain 69m 26s) Loss: 0.0455(0.1163) Grad: 1.3897  LR: 0.00000483  \n",
    "Epoch: [4][29000/78269] Elapsed 40m 3s (remain 68m 3s) Loss: 0.0386(0.1163) Grad: 1.1915  LR: 0.00000479  \n",
    "Epoch: [4][30000/78269] Elapsed 41m 25s (remain 66m 39s) Loss: 0.1321(0.1164) Grad: 4.3093  LR: 0.00000476  \n",
    "Epoch: [4][31000/78269] Elapsed 42m 47s (remain 65m 14s) Loss: 0.1538(0.1164) Grad: 9.0142  LR: 0.00000472  \n",
    "Epoch: [4][32000/78269] Elapsed 44m 9s (remain 63m 51s) Loss: 0.0631(0.1165) Grad: 3.5223  LR: 0.00000468  \n",
    "Epoch: [4][33000/78269] Elapsed 45m 31s (remain 62m 27s) Loss: 0.1877(0.1167) Grad: 9.4599  LR: 0.00000465  \n",
    "Epoch: [4][34000/78269] Elapsed 46m 54s (remain 61m 3s) Loss: 0.0270(0.1168) Grad: 1.7469  LR: 0.00000461  \n",
    "Epoch: [4][35000/78269] Elapsed 48m 16s (remain 59m 40s) Loss: 0.1180(0.1167) Grad: 3.1831  LR: 0.00000457  \n",
    "Epoch: [4][36000/78269] Elapsed 49m 38s (remain 58m 16s) Loss: 0.1168(0.1168) Grad: 16.0367  LR: 0.00000454  \n",
    "Epoch: [4][37000/78269] Elapsed 51m 1s (remain 56m 54s) Loss: 0.1324(0.1169) Grad: 5.3015  LR: 0.00000450  \n",
    "Epoch: [4][38000/78269] Elapsed 52m 23s (remain 55m 31s) Loss: 0.0721(0.1169) Grad: 16.5087  LR: 0.00000446  \n",
    "Epoch: [4][39000/78269] Elapsed 53m 45s (remain 54m 7s) Loss: 0.0794(0.1169) Grad: 2.8116  LR: 0.00000442  \n",
    "Epoch: [4][40000/78269] Elapsed 55m 8s (remain 52m 44s) Loss: 0.0866(0.1167) Grad: 2.2142  LR: 0.00000439  \n",
    "Epoch: [4][41000/78269] Elapsed 56m 30s (remain 51m 21s) Loss: 0.0160(0.1169) Grad: 1.3203  LR: 0.00000435  \n",
    "Epoch: [4][42000/78269] Elapsed 57m 51s (remain 49m 58s) Loss: 0.0069(0.1168) Grad: 1.0643  LR: 0.00000431  \n",
    "Epoch: [4][43000/78269] Elapsed 59m 14s (remain 48m 35s) Loss: 0.0871(0.1168) Grad: 3.2210  LR: 0.00000428  \n",
    "Epoch: [4][44000/78269] Elapsed 60m 36s (remain 47m 12s) Loss: 0.0576(0.1167) Grad: 1.3671  LR: 0.00000424  \n",
    "Epoch: [4][45000/78269] Elapsed 61m 58s (remain 45m 49s) Loss: 0.0236(0.1167) Grad: 1.7502  LR: 0.00000420  \n",
    "Epoch: [4][46000/78269] Elapsed 63m 21s (remain 44m 26s) Loss: 0.3283(0.1167) Grad: 15.1705  LR: 0.00000417  \n",
    "Epoch: [4][47000/78269] Elapsed 64m 44s (remain 43m 3s) Loss: 0.0154(0.1167) Grad: 1.3577  LR: 0.00000413  \n",
    "Epoch: [4][48000/78269] Elapsed 66m 6s (remain 41m 41s) Loss: 0.0192(0.1167) Grad: 1.2634  LR: 0.00000409  \n",
    "Epoch: [4][49000/78269] Elapsed 67m 28s (remain 40m 18s) Loss: 0.0289(0.1167) Grad: 1.2232  LR: 0.00000406  \n",
    "Epoch: [4][50000/78269] Elapsed 68m 50s (remain 38m 55s) Loss: 0.1753(0.1168) Grad: 2.5455  LR: 0.00000402  \n",
    "Epoch: [4][51000/78269] Elapsed 70m 13s (remain 37m 32s) Loss: 0.0011(0.1168) Grad: 0.0544  LR: 0.00000398  \n",
    "Epoch: [4][52000/78269] Elapsed 71m 36s (remain 36m 10s) Loss: 0.1583(0.1168) Grad: 5.1064  LR: 0.00000395  \n",
    "Epoch: [4][53000/78269] Elapsed 72m 58s (remain 34m 47s) Loss: 0.0352(0.1168) Grad: 1.2604  LR: 0.00000391  \n",
    "Epoch: [4][54000/78269] Elapsed 74m 20s (remain 33m 24s) Loss: 0.2317(0.1168) Grad: 10.0635  LR: 0.00000388  \n",
    "Epoch: [4][55000/78269] Elapsed 75m 42s (remain 32m 1s) Loss: 0.1634(0.1167) Grad: 7.1916  LR: 0.00000384  \n",
    "Epoch: [4][56000/78269] Elapsed 77m 5s (remain 30m 39s) Loss: 0.1642(0.1167) Grad: 9.6656  LR: 0.00000380  \n",
    "Epoch: [4][57000/78269] Elapsed 78m 26s (remain 29m 16s) Loss: 0.0357(0.1167) Grad: 4.3991  LR: 0.00000377  \n",
    "Epoch: [4][58000/78269] Elapsed 79m 49s (remain 27m 53s) Loss: 0.0175(0.1167) Grad: 0.9953  LR: 0.00000373  \n",
    "Epoch: [4][59000/78269] Elapsed 81m 11s (remain 26m 30s) Loss: 0.2502(0.1166) Grad: 6.3837  LR: 0.00000370  \n",
    "Epoch: [4][60000/78269] Elapsed 82m 33s (remain 25m 8s) Loss: 0.1418(0.1166) Grad: 6.7617  LR: 0.00000366  \n",
    "Epoch: [4][61000/78269] Elapsed 83m 55s (remain 23m 45s) Loss: 0.1070(0.1166) Grad: 9.0369  LR: 0.00000362  \n",
    "Epoch: [4][62000/78269] Elapsed 85m 17s (remain 22m 22s) Loss: 0.0500(0.1166) Grad: 3.1001  LR: 0.00000359  \n",
    "Epoch: [4][63000/78269] Elapsed 86m 39s (remain 21m 0s) Loss: 0.1183(0.1165) Grad: 17.1415  LR: 0.00000355  \n",
    "Epoch: [4][64000/78269] Elapsed 88m 1s (remain 19m 37s) Loss: 0.4003(0.1165) Grad: 9.2638  LR: 0.00000352  \n",
    "Epoch: [4][65000/78269] Elapsed 89m 23s (remain 18m 14s) Loss: 0.0507(0.1165) Grad: 3.3327  LR: 0.00000348  \n",
    "Epoch: [4][66000/78269] Elapsed 90m 45s (remain 16m 52s) Loss: 0.0968(0.1165) Grad: 12.0742  LR: 0.00000345  \n",
    "Epoch: [4][67000/78269] Elapsed 92m 7s (remain 15m 29s) Loss: 0.1324(0.1164) Grad: 11.8265  LR: 0.00000341  \n",
    "Epoch: [4][68000/78269] Elapsed 93m 29s (remain 14m 7s) Loss: 0.0856(0.1164) Grad: 9.9422  LR: 0.00000338  \n",
    "Epoch: [4][69000/78269] Elapsed 94m 53s (remain 12m 44s) Loss: 0.2066(0.1165) Grad: 3.3257  LR: 0.00000334  \n",
    "Epoch: [4][70000/78269] Elapsed 96m 15s (remain 11m 22s) Loss: 0.1440(0.1165) Grad: 5.9186  LR: 0.00000331  \n",
    "Epoch: [4][71000/78269] Elapsed 97m 37s (remain 9m 59s) Loss: 0.1999(0.1165) Grad: 9.5694  LR: 0.00000327  \n",
    "Epoch: [4][72000/78269] Elapsed 98m 59s (remain 8m 37s) Loss: 0.1547(0.1165) Grad: 4.5974  LR: 0.00000324  \n",
    "Epoch: [4][73000/78269] Elapsed 100m 22s (remain 7m 14s) Loss: 0.1118(0.1165) Grad: 2.4996  LR: 0.00000320  \n",
    "Epoch: [4][74000/78269] Elapsed 101m 44s (remain 5m 52s) Loss: 0.0766(0.1165) Grad: 1.8763  LR: 0.00000317  \n",
    "Epoch: [4][75000/78269] Elapsed 103m 6s (remain 4m 29s) Loss: 0.2030(0.1165) Grad: 8.3310  LR: 0.00000313  \n",
    "Epoch: [4][76000/78269] Elapsed 104m 28s (remain 3m 7s) Loss: 0.1206(0.1164) Grad: 7.0817  LR: 0.00000310  \n",
    "Epoch: [4][77000/78269] Elapsed 105m 50s (remain 1m 44s) Loss: 0.0878(0.1164) Grad: 7.2147  LR: 0.00000306  \n",
    "Epoch: [4][78000/78269] Elapsed 107m 12s (remain 0m 22s) Loss: 0.1705(0.1164) Grad: 8.9395  LR: 0.00000303  \n",
    "Epoch: [4][78268/78269] Elapsed 107m 34s (remain 0m 0s) Loss: 0.2606(0.1164) Grad: 7.8040  LR: 0.00000302  \n",
    "EVAL: [0/19225] Elapsed 0m 1s (remain 349m 47s) Loss: 0.0035(0.0035) \n",
    "EVAL: [1000/19225] Elapsed 0m 13s (remain 4m 0s) Loss: 0.7636(0.2184) \n",
    "EVAL: [2000/19225] Elapsed 0m 24s (remain 3m 32s) Loss: 0.0000(0.1861) \n",
    "EVAL: [3000/19225] Elapsed 0m 35s (remain 3m 9s) Loss: 0.0000(0.1494) \n",
    "EVAL: [4000/19225] Elapsed 0m 47s (remain 3m 2s) Loss: 0.0965(0.1407) \n",
    "EVAL: [5000/19225] Elapsed 1m 1s (remain 2m 54s) Loss: 0.0057(0.1522) \n",
    "EVAL: [6000/19225] Elapsed 1m 15s (remain 2m 46s) Loss: 0.2783(0.1587) \n",
    "EVAL: [7000/19225] Elapsed 1m 29s (remain 2m 36s) Loss: 0.0069(0.1632) \n",
    "EVAL: [8000/19225] Elapsed 1m 44s (remain 2m 25s) Loss: 0.0020(0.1696) \n",
    "EVAL: [9000/19225] Elapsed 1m 58s (remain 2m 14s) Loss: 0.0025(0.1763) \n",
    "EVAL: [10000/19225] Elapsed 2m 13s (remain 2m 3s) Loss: 0.0316(0.1787) \n",
    "EVAL: [11000/19225] Elapsed 2m 29s (remain 1m 51s) Loss: 0.1695(0.1798) \n",
    "EVAL: [12000/19225] Elapsed 2m 44s (remain 1m 39s) Loss: 0.0010(0.1830) \n",
    "EVAL: [13000/19225] Elapsed 3m 0s (remain 1m 26s) Loss: 0.0066(0.1846) \n",
    "EVAL: [14000/19225] Elapsed 3m 17s (remain 1m 13s) Loss: 0.0012(0.1837) \n",
    "EVAL: [15000/19225] Elapsed 3m 33s (remain 1m 0s) Loss: 0.0079(0.1849) \n",
    "EVAL: [16000/19225] Elapsed 3m 51s (remain 0m 46s) Loss: 0.5376(0.1854) \n",
    "EVAL: [17000/19225] Elapsed 4m 9s (remain 0m 32s) Loss: 0.0253(0.1849) \n",
    "EVAL: [18000/19225] Elapsed 4m 27s (remain 0m 18s) Loss: 0.0088(0.1838) \n",
    "EVAL: [19000/19225] Elapsed 4m 48s (remain 0m 3s) Loss: 0.9802(0.1832) \n",
    "EVAL: [19224/19225] Elapsed 4m 54s (remain 0m 0s) Loss: 0.1449(0.1830) \n",
    "Epoch 4 - avg_train_loss: 0.1164  avg_val_loss: 0.1830  time: 7006s\n",
    "Epoch 4 - Score: 0.4857 - Threshold: 0.03700\n",
    "Epoch 4 - Save Best Score: 0.4857 Model\n",
    "Epoch: [5][0/78269] Elapsed 0m 1s (remain 1536m 37s) Loss: 0.1153(0.1153) Grad: 6.0139  LR: 0.00000302  \n",
    "Epoch: [5][1000/78269] Elapsed 1m 23s (remain 107m 5s) Loss: 0.0450(0.0969) Grad: 2.7287  LR: 0.00000299  \n",
    "Epoch: [5][2000/78269] Elapsed 2m 45s (remain 104m 55s) Loss: 0.1526(0.0951) Grad: 9.4190  LR: 0.00000295  \n",
    "Epoch: [5][3000/78269] Elapsed 4m 7s (remain 103m 20s) Loss: 0.6710(0.0981) Grad: 25.0426  LR: 0.00000292  \n",
    "Epoch: [5][4000/78269] Elapsed 5m 29s (remain 101m 54s) Loss: 0.0444(0.0993) Grad: 4.0984  LR: 0.00000288  \n",
    "Epoch: [5][5000/78269] Elapsed 6m 51s (remain 100m 28s) Loss: 0.0101(0.1014) Grad: 1.5684  LR: 0.00000285  \n",
    "Epoch: [5][6000/78269] Elapsed 8m 13s (remain 99m 2s) Loss: 0.0068(0.1016) Grad: 0.3127  LR: 0.00000282  \n",
    "Epoch: [5][7000/78269] Elapsed 9m 35s (remain 97m 38s) Loss: 0.1396(0.1019) Grad: 5.1966  LR: 0.00000278  \n",
    "Epoch: [5][8000/78269] Elapsed 10m 57s (remain 96m 17s) Loss: 0.2667(0.1028) Grad: 37.3245  LR: 0.00000275  \n",
    "Epoch: [5][9000/78269] Elapsed 12m 19s (remain 94m 54s) Loss: 0.1562(0.1030) Grad: 13.7763  LR: 0.00000272  \n",
    "Epoch: [5][10000/78269] Elapsed 13m 42s (remain 93m 32s) Loss: 0.0722(0.1039) Grad: 6.3934  LR: 0.00000268  \n",
    "Epoch: [5][11000/78269] Elapsed 15m 3s (remain 92m 7s) Loss: 0.2104(0.1040) Grad: 13.5479  LR: 0.00000265  \n",
    "Epoch: [5][12000/78269] Elapsed 16m 26s (remain 90m 45s) Loss: 0.1643(0.1041) Grad: 8.7678  LR: 0.00000262  \n",
    "Epoch: [5][13000/78269] Elapsed 17m 48s (remain 89m 22s) Loss: 0.2269(0.1039) Grad: 6.5629  LR: 0.00000259  \n",
    "Epoch: [5][14000/78269] Elapsed 19m 10s (remain 87m 58s) Loss: 0.1639(0.1041) Grad: 17.9874  LR: 0.00000255  \n",
    "Epoch: [5][15000/78269] Elapsed 20m 32s (remain 86m 37s) Loss: 0.0052(0.1042) Grad: 0.6602  LR: 0.00000252  \n",
    "Epoch: [5][16000/78269] Elapsed 21m 54s (remain 85m 14s) Loss: 0.0215(0.1047) Grad: 2.5555  LR: 0.00000249  \n",
    "Epoch: [5][17000/78269] Elapsed 23m 16s (remain 83m 51s) Loss: 0.0110(0.1045) Grad: 1.0891  LR: 0.00000246  \n",
    "Epoch: [5][18000/78269] Elapsed 24m 38s (remain 82m 29s) Loss: 0.3395(0.1045) Grad: 11.1527  LR: 0.00000242  \n",
    "Epoch: [5][19000/78269] Elapsed 26m 1s (remain 81m 9s) Loss: 0.0051(0.1046) Grad: 0.8798  LR: 0.00000239  \n",
    "Epoch: [5][20000/78269] Elapsed 27m 24s (remain 79m 49s) Loss: 0.0073(0.1045) Grad: 0.4446  LR: 0.00000236  \n",
    "Epoch: [5][21000/78269] Elapsed 28m 47s (remain 78m 31s) Loss: 0.0395(0.1042) Grad: 1.9581  LR: 0.00000233  \n",
    "Epoch: [5][22000/78269] Elapsed 30m 11s (remain 77m 12s) Loss: 0.0006(0.1044) Grad: 0.0566  LR: 0.00000230  \n",
    "Epoch: [5][23000/78269] Elapsed 31m 34s (remain 75m 52s) Loss: 0.2185(0.1041) Grad: 8.1562  LR: 0.00000227  \n",
    "Epoch: [5][24000/78269] Elapsed 32m 57s (remain 74m 31s) Loss: 0.0028(0.1043) Grad: 0.3289  LR: 0.00000224  \n",
    "Epoch: [5][25000/78269] Elapsed 34m 20s (remain 73m 11s) Loss: 0.0042(0.1042) Grad: 0.4896  LR: 0.00000221  \n",
    "Epoch: [5][26000/78269] Elapsed 35m 44s (remain 71m 50s) Loss: 0.0031(0.1044) Grad: 1.0223  LR: 0.00000217  \n",
    "Epoch: [5][27000/78269] Elapsed 37m 7s (remain 70m 30s) Loss: 0.0305(0.1044) Grad: 3.7406  LR: 0.00000214  \n",
    "Epoch: [5][28000/78269] Elapsed 38m 31s (remain 69m 9s) Loss: 0.0290(0.1045) Grad: 1.5718  LR: 0.00000211  \n",
    "Epoch: [5][29000/78269] Elapsed 39m 54s (remain 67m 48s) Loss: 0.0609(0.1046) Grad: 4.8479  LR: 0.00000208  \n",
    "Epoch: [5][30000/78269] Elapsed 41m 18s (remain 66m 26s) Loss: 0.0009(0.1050) Grad: 0.1091  LR: 0.00000205  \n",
    "Epoch: [5][31000/78269] Elapsed 42m 41s (remain 65m 5s) Loss: 0.0761(0.1049) Grad: 14.9728  LR: 0.00000202  \n",
    "Epoch: [5][32000/78269] Elapsed 44m 4s (remain 63m 43s) Loss: 0.0102(0.1049) Grad: 1.9177  LR: 0.00000199  \n",
    "Epoch: [5][33000/78269] Elapsed 45m 28s (remain 62m 22s) Loss: 0.0020(0.1050) Grad: 0.3353  LR: 0.00000196  \n",
    "Epoch: [5][34000/78269] Elapsed 46m 51s (remain 61m 1s) Loss: 0.0553(0.1051) Grad: 3.4040  LR: 0.00000193  \n",
    "Epoch: [5][35000/78269] Elapsed 48m 16s (remain 59m 40s) Loss: 0.1996(0.1051) Grad: 8.5004  LR: 0.00000191  \n",
    "Epoch: [5][36000/78269] Elapsed 49m 39s (remain 58m 17s) Loss: 0.3101(0.1050) Grad: 27.7870  LR: 0.00000188  \n",
    "Epoch: [5][37000/78269] Elapsed 51m 2s (remain 56m 55s) Loss: 0.3348(0.1050) Grad: 12.6695  LR: 0.00000185  \n",
    "Epoch: [5][38000/78269] Elapsed 52m 25s (remain 55m 33s) Loss: 0.0026(0.1050) Grad: 0.2642  LR: 0.00000182  \n",
    "Epoch: [5][39000/78269] Elapsed 53m 48s (remain 54m 10s) Loss: 0.2682(0.1049) Grad: 29.3156  LR: 0.00000179  \n",
    "Epoch: [5][40000/78269] Elapsed 55m 10s (remain 52m 46s) Loss: 0.0273(0.1049) Grad: 2.5182  LR: 0.00000176  \n",
    "Epoch: [5][41000/78269] Elapsed 56m 32s (remain 51m 23s) Loss: 0.0897(0.1050) Grad: 5.5454  LR: 0.00000173  \n",
    "Epoch: [5][42000/78269] Elapsed 57m 54s (remain 50m 0s) Loss: 0.3062(0.1050) Grad: 10.6544  LR: 0.00000171  \n",
    "Epoch: [5][43000/78269] Elapsed 59m 16s (remain 48m 37s) Loss: 0.0037(0.1051) Grad: 0.3295  LR: 0.00000168  \n",
    "Epoch: [5][44000/78269] Elapsed 60m 39s (remain 47m 14s) Loss: 0.0979(0.1051) Grad: 8.1895  LR: 0.00000165  \n",
    "Epoch: [5][45000/78269] Elapsed 62m 1s (remain 45m 51s) Loss: 0.1088(0.1051) Grad: 39.3498  LR: 0.00000162  \n",
    "Epoch: [5][46000/78269] Elapsed 63m 23s (remain 44m 28s) Loss: 0.1440(0.1051) Grad: 11.0786  LR: 0.00000159  \n",
    "Epoch: [5][47000/78269] Elapsed 64m 45s (remain 43m 5s) Loss: 0.0852(0.1050) Grad: 6.7354  LR: 0.00000157  \n",
    "Epoch: [5][48000/78269] Elapsed 66m 7s (remain 41m 41s) Loss: 0.0055(0.1051) Grad: 0.6591  LR: 0.00000154  \n",
    "Epoch: [5][49000/78269] Elapsed 67m 29s (remain 40m 18s) Loss: 0.0846(0.1051) Grad: 5.3233  LR: 0.00000151  \n",
    "Epoch: [5][50000/78269] Elapsed 68m 51s (remain 38m 55s) Loss: 0.1464(0.1052) Grad: 13.5174  LR: 0.00000149  \n",
    "Epoch: [5][51000/78269] Elapsed 70m 14s (remain 37m 33s) Loss: 0.0469(0.1052) Grad: 16.9718  LR: 0.00000146  \n",
    "Epoch: [5][52000/78269] Elapsed 71m 36s (remain 36m 10s) Loss: 0.0005(0.1052) Grad: 0.0724  LR: 0.00000144  \n",
    "Epoch: [5][53000/78269] Elapsed 72m 58s (remain 34m 47s) Loss: 0.0143(0.1053) Grad: 0.7911  LR: 0.00000141  \n",
    "Epoch: [5][54000/78269] Elapsed 74m 20s (remain 33m 24s) Loss: 0.0025(0.1054) Grad: 0.1830  LR: 0.00000138  \n",
    "Epoch: [5][55000/78269] Elapsed 75m 42s (remain 32m 1s) Loss: 0.1263(0.1054) Grad: 7.0805  LR: 0.00000136  \n",
    "Epoch: [5][56000/78269] Elapsed 77m 4s (remain 30m 38s) Loss: 0.5877(0.1054) Grad: 17.5836  LR: 0.00000133  \n",
    "Epoch: [5][57000/78269] Elapsed 78m 27s (remain 29m 16s) Loss: 0.0047(0.1053) Grad: 0.6484  LR: 0.00000131  \n",
    "Epoch: [5][58000/78269] Elapsed 79m 49s (remain 27m 53s) Loss: 0.1924(0.1053) Grad: 25.1076  LR: 0.00000128  \n",
    "Epoch: [5][59000/78269] Elapsed 81m 11s (remain 26m 30s) Loss: 0.0955(0.1053) Grad: 8.4341  LR: 0.00000126  \n",
    "Epoch: [5][60000/78269] Elapsed 82m 33s (remain 25m 8s) Loss: 0.2105(0.1052) Grad: 3.7268  LR: 0.00000123  \n",
    "Epoch: [5][61000/78269] Elapsed 83m 56s (remain 23m 45s) Loss: 0.1870(0.1052) Grad: 13.9925  LR: 0.00000121  \n",
    "Epoch: [5][62000/78269] Elapsed 85m 18s (remain 22m 23s) Loss: 0.2326(0.1052) Grad: 5.3898  LR: 0.00000118  \n",
    "Epoch: [5][63000/78269] Elapsed 86m 40s (remain 21m 0s) Loss: 0.0160(0.1052) Grad: 13.7896  LR: 0.00000116  \n",
    "Epoch: [5][64000/78269] Elapsed 88m 3s (remain 19m 37s) Loss: 0.0005(0.1052) Grad: 0.0518  LR: 0.00000114  \n",
    "Epoch: [5][65000/78269] Elapsed 89m 25s (remain 18m 15s) Loss: 0.0254(0.1052) Grad: 2.0818  LR: 0.00000111  \n",
    "Epoch: [5][66000/78269] Elapsed 90m 47s (remain 16m 52s) Loss: 0.0038(0.1051) Grad: 0.3509  LR: 0.00000109  \n",
    "Epoch: [5][67000/78269] Elapsed 92m 9s (remain 15m 29s) Loss: 0.0359(0.1051) Grad: 5.1266  LR: 0.00000107  \n",
    "Epoch: [5][68000/78269] Elapsed 93m 31s (remain 14m 7s) Loss: 0.2468(0.1051) Grad: 13.8404  LR: 0.00000104  \n",
    "Epoch: [5][69000/78269] Elapsed 94m 53s (remain 12m 44s) Loss: 0.0005(0.1051) Grad: 0.0594  LR: 0.00000102  \n",
    "Epoch: [5][70000/78269] Elapsed 96m 15s (remain 11m 22s) Loss: 0.0411(0.1052) Grad: 31.5831  LR: 0.00000100  \n",
    "Epoch: [5][71000/78269] Elapsed 97m 37s (remain 9m 59s) Loss: 0.0253(0.1052) Grad: 1.0145  LR: 0.00000098  \n",
    "Epoch: [5][72000/78269] Elapsed 98m 59s (remain 8m 37s) Loss: 0.0441(0.1051) Grad: 9.5842  LR: 0.00000096  \n",
    "Epoch: [5][73000/78269] Elapsed 100m 21s (remain 7m 14s) Loss: 0.0007(0.1051) Grad: 0.0417  LR: 0.00000093  \n",
    "Epoch: [5][74000/78269] Elapsed 101m 43s (remain 5m 52s) Loss: 0.2893(0.1052) Grad: 7.0211  LR: 0.00000091  \n",
    "Epoch: [5][75000/78269] Elapsed 103m 5s (remain 4m 29s) Loss: 0.0167(0.1051) Grad: 0.8156  LR: 0.00000089  \n",
    "Epoch: [5][76000/78269] Elapsed 104m 27s (remain 3m 7s) Loss: 0.0971(0.1051) Grad: 6.3094  LR: 0.00000087  \n",
    "Epoch: [5][77000/78269] Elapsed 105m 49s (remain 1m 44s) Loss: 0.5003(0.1052) Grad: 12.4228  LR: 0.00000085  \n",
    "Epoch: [5][78000/78269] Elapsed 107m 12s (remain 0m 22s) Loss: 0.3080(0.1052) Grad: 22.8545  LR: 0.00000083  \n",
    "Epoch: [5][78268/78269] Elapsed 107m 34s (remain 0m 0s) Loss: 0.0424(0.1052) Grad: 2.3095  LR: 0.00000082  \n",
    "EVAL: [0/19225] Elapsed 0m 1s (remain 326m 18s) Loss: 0.0003(0.0003) \n",
    "EVAL: [1000/19225] Elapsed 0m 13s (remain 3m 59s) Loss: 1.1442(0.3031) \n",
    "EVAL: [2000/19225] Elapsed 0m 24s (remain 3m 32s) Loss: 0.0000(0.2527) \n",
    "EVAL: [3000/19225] Elapsed 0m 35s (remain 3m 9s) Loss: 0.0000(0.2022) \n",
    "EVAL: [4000/19225] Elapsed 0m 47s (remain 3m 2s) Loss: 0.1137(0.1916) \n",
    "EVAL: [5000/19225] Elapsed 1m 1s (remain 2m 54s) Loss: 0.0005(0.2088) \n",
    "EVAL: [6000/19225] Elapsed 1m 15s (remain 2m 46s) Loss: 0.4788(0.2164) \n",
    "EVAL: [7000/19225] Elapsed 1m 29s (remain 2m 36s) Loss: 0.0678(0.2237) \n",
    "EVAL: [8000/19225] Elapsed 1m 44s (remain 2m 26s) Loss: 0.0151(0.2334) \n",
    "EVAL: [9000/19225] Elapsed 1m 58s (remain 2m 15s) Loss: 0.0001(0.2425) \n",
    "EVAL: [10000/19225] Elapsed 2m 14s (remain 2m 3s) Loss: 0.0210(0.2465) \n",
    "EVAL: [11000/19225] Elapsed 2m 29s (remain 1m 51s) Loss: 0.3022(0.2483) \n",
    "EVAL: [12000/19225] Elapsed 2m 44s (remain 1m 39s) Loss: 0.0001(0.2520) \n",
    "EVAL: [13000/19225] Elapsed 3m 0s (remain 1m 26s) Loss: 0.0008(0.2547) \n",
    "EVAL: [14000/19225] Elapsed 3m 17s (remain 1m 13s) Loss: 0.0001(0.2534) \n",
    "EVAL: [15000/19225] Elapsed 3m 33s (remain 1m 0s) Loss: 0.0019(0.2551) \n",
    "EVAL: [16000/19225] Elapsed 3m 51s (remain 0m 46s) Loss: 0.8604(0.2558) \n",
    "EVAL: [17000/19225] Elapsed 4m 9s (remain 0m 32s) Loss: 0.0044(0.2549) \n",
    "EVAL: [18000/19225] Elapsed 4m 27s (remain 0m 18s) Loss: 0.0026(0.2534) \n",
    "EVAL: [19000/19225] Elapsed 4m 48s (remain 0m 3s) Loss: 1.5066(0.2529) \n",
    "EVAL: [19224/19225] Elapsed 4m 54s (remain 0m 0s) Loss: 0.0455(0.2532) \n",
    "Epoch 5 - avg_train_loss: 0.1052  avg_val_loss: 0.2532  time: 7013s\n",
    "Epoch 5 - Score: 0.4893 - Threshold: 0.01000\n",
    "Epoch 5 - Save Best Score: 0.4893 Model\n",
    "Epoch: [6][0/78269] Elapsed 0m 1s (remain 1633m 56s) Loss: 0.0172(0.0172) Grad: 4.8524  LR: 0.00000082  \n",
    "Epoch: [6][1000/78269] Elapsed 1m 23s (remain 107m 55s) Loss: 0.0008(0.0882) Grad: 0.1213  LR: 0.00000080  \n",
    "Epoch: [6][2000/78269] Elapsed 2m 46s (remain 105m 48s) Loss: 0.1588(0.0919) Grad: 11.8090  LR: 0.00000078  \n",
    "Epoch: [6][3000/78269] Elapsed 4m 8s (remain 104m 0s) Loss: 0.0417(0.0923) Grad: 7.6065  LR: 0.00000076  \n",
    "Epoch: [6][4000/78269] Elapsed 5m 30s (remain 102m 19s) Loss: 0.2161(0.0947) Grad: 4.7783  LR: 0.00000074  \n",
    "Epoch: [6][5000/78269] Elapsed 6m 52s (remain 100m 50s) Loss: 0.1747(0.0958) Grad: 11.8670  LR: 0.00000072  \n",
    "Epoch: [6][6000/78269] Elapsed 8m 15s (remain 99m 21s) Loss: 0.0029(0.0969) Grad: 0.7641  LR: 0.00000070  \n",
    "Epoch: [6][7000/78269] Elapsed 9m 36s (remain 97m 53s) Loss: 0.0124(0.0981) Grad: 5.6094  LR: 0.00000069  \n",
    "Epoch: [6][8000/78269] Elapsed 10m 59s (remain 96m 28s) Loss: 0.1377(0.0977) Grad: 15.5789  LR: 0.00000067  \n",
    "Epoch: [6][9000/78269] Elapsed 12m 21s (remain 95m 3s) Loss: 0.2864(0.0979) Grad: 11.8884  LR: 0.00000065  \n",
    "Epoch: [6][10000/78269] Elapsed 13m 43s (remain 93m 39s) Loss: 0.2581(0.0982) Grad: 10.8977  LR: 0.00000063  \n",
    "Epoch: [6][11000/78269] Elapsed 15m 5s (remain 92m 16s) Loss: 0.0055(0.0986) Grad: 3.0285  LR: 0.00000061  \n",
    "Epoch: [6][12000/78269] Elapsed 16m 27s (remain 90m 51s) Loss: 0.0514(0.0988) Grad: 22.5642  LR: 0.00000059  \n",
    "Epoch: [6][13000/78269] Elapsed 17m 49s (remain 89m 30s) Loss: 0.0884(0.0993) Grad: 14.8080  LR: 0.00000058  \n",
    "Epoch: [6][14000/78269] Elapsed 19m 11s (remain 88m 7s) Loss: 0.0980(0.0990) Grad: 10.7209  LR: 0.00000056  \n",
    "Epoch: [6][15000/78269] Elapsed 20m 34s (remain 86m 45s) Loss: 0.0444(0.0994) Grad: 11.5517  LR: 0.00000054  \n",
    "Epoch: [6][16000/78269] Elapsed 21m 56s (remain 85m 21s) Loss: 0.0001(0.0998) Grad: 0.0036  LR: 0.00000053  \n",
    "Epoch: [6][17000/78269] Elapsed 23m 18s (remain 83m 58s) Loss: 0.3638(0.1003) Grad: 9.9523  LR: 0.00000051  \n",
    "Epoch: [6][18000/78269] Elapsed 24m 39s (remain 82m 35s) Loss: 0.0143(0.1007) Grad: 2.9004  LR: 0.00000049  \n",
    "Epoch: [6][19000/78269] Elapsed 26m 2s (remain 81m 12s) Loss: 0.0081(0.1008) Grad: 3.5322  LR: 0.00000048  \n",
    "Epoch: [6][20000/78269] Elapsed 27m 24s (remain 79m 50s) Loss: 0.0005(0.1012) Grad: 0.0410  LR: 0.00000046  \n",
    "Epoch: [6][21000/78269] Elapsed 28m 47s (remain 78m 31s) Loss: 0.1814(0.1012) Grad: 19.9241  LR: 0.00000045  \n",
    "Epoch: [6][22000/78269] Elapsed 30m 12s (remain 77m 16s) Loss: 0.0008(0.1012) Grad: 0.1451  LR: 0.00000043  \n",
    "Epoch: [6][23000/78269] Elapsed 31m 37s (remain 75m 58s) Loss: 0.0176(0.1016) Grad: 13.2058  LR: 0.00000042  \n",
    "Epoch: [6][24000/78269] Elapsed 33m 3s (remain 74m 44s) Loss: 0.1447(0.1015) Grad: 14.2226  LR: 0.00000040  \n",
    "Epoch: [6][25000/78269] Elapsed 34m 30s (remain 73m 31s) Loss: 0.0129(0.1015) Grad: 1.5069  LR: 0.00000039  \n",
    "Epoch: [6][26000/78269] Elapsed 35m 53s (remain 72m 9s) Loss: 0.4651(0.1017) Grad: 13.4672  LR: 0.00000037  \n",
    "Epoch: [6][27000/78269] Elapsed 37m 17s (remain 70m 48s) Loss: 0.1195(0.1020) Grad: 54.6208  LR: 0.00000036  \n",
    "Epoch: [6][28000/78269] Elapsed 38m 40s (remain 69m 26s) Loss: 0.1052(0.1020) Grad: 7.0461  LR: 0.00000035  \n",
    "Epoch: [6][29000/78269] Elapsed 40m 4s (remain 68m 4s) Loss: 0.0265(0.1022) Grad: 9.0796  LR: 0.00000033  \n",
    "Epoch: [6][30000/78269] Elapsed 41m 27s (remain 66m 41s) Loss: 0.0004(0.1023) Grad: 0.0806  LR: 0.00000032  \n",
    "Epoch: [6][31000/78269] Elapsed 42m 50s (remain 65m 19s) Loss: 0.0028(0.1023) Grad: 0.1901  LR: 0.00000031  \n",
    "Epoch: [6][32000/78269] Elapsed 44m 14s (remain 63m 57s) Loss: 0.0031(0.1023) Grad: 0.5145  LR: 0.00000029  \n",
    "Epoch: [6][33000/78269] Elapsed 45m 37s (remain 62m 35s) Loss: 0.1399(0.1023) Grad: 6.9382  LR: 0.00000028  \n",
    "Epoch: [6][34000/78269] Elapsed 47m 1s (remain 61m 12s) Loss: 0.0112(0.1025) Grad: 1.5765  LR: 0.00000027  \n",
    "Epoch: [6][35000/78269] Elapsed 48m 24s (remain 59m 50s) Loss: 0.2363(0.1025) Grad: 3.0355  LR: 0.00000026  \n",
    "Epoch: [6][36000/78269] Elapsed 49m 47s (remain 58m 27s) Loss: 0.0809(0.1025) Grad: 15.5139  LR: 0.00000024  \n",
    "Epoch: [6][37000/78269] Elapsed 51m 11s (remain 57m 5s) Loss: 0.1013(0.1025) Grad: 12.5861  LR: 0.00000023  \n",
    "Epoch: [6][38000/78269] Elapsed 52m 34s (remain 55m 43s) Loss: 0.0189(0.1027) Grad: 11.2407  LR: 0.00000022  \n",
    "Epoch: [6][39000/78269] Elapsed 53m 58s (remain 54m 20s) Loss: 0.0019(0.1027) Grad: 0.3571  LR: 0.00000021  \n",
    "Epoch: [6][40000/78269] Elapsed 55m 21s (remain 52m 57s) Loss: 0.0015(0.1027) Grad: 1.1053  LR: 0.00000020  \n",
    "Epoch: [6][41000/78269] Elapsed 56m 45s (remain 51m 35s) Loss: 0.0003(0.1028) Grad: 0.0504  LR: 0.00000019  \n",
    "Epoch: [6][42000/78269] Elapsed 58m 8s (remain 50m 12s) Loss: 0.3783(0.1027) Grad: 20.1644  LR: 0.00000018  \n",
    "Epoch: [6][43000/78269] Elapsed 59m 31s (remain 48m 49s) Loss: 0.0008(0.1027) Grad: 0.1719  LR: 0.00000017  \n",
    "Epoch: [6][44000/78269] Elapsed 60m 55s (remain 47m 26s) Loss: 0.3284(0.1029) Grad: 7.6233  LR: 0.00000016  \n",
    "Epoch: [6][45000/78269] Elapsed 62m 18s (remain 46m 3s) Loss: 0.0001(0.1030) Grad: 0.0058  LR: 0.00000015  \n",
    "Epoch: [6][46000/78269] Elapsed 63m 41s (remain 44m 40s) Loss: 0.1920(0.1031) Grad: 6.4039  LR: 0.00000014  \n",
    "Epoch: [6][47000/78269] Elapsed 65m 5s (remain 43m 17s) Loss: 0.0002(0.1032) Grad: 0.0434  LR: 0.00000013  \n",
    "Epoch: [6][48000/78269] Elapsed 66m 28s (remain 41m 55s) Loss: 0.0002(0.1032) Grad: 0.0136  LR: 0.00000013  \n",
    "Epoch: [6][49000/78269] Elapsed 67m 51s (remain 40m 32s) Loss: 0.2572(0.1032) Grad: 5.9379  LR: 0.00000012  \n",
    "Epoch: [6][50000/78269] Elapsed 69m 15s (remain 39m 9s) Loss: 0.0251(0.1032) Grad: 2.5216  LR: 0.00000011  \n",
    "Epoch: [6][51000/78269] Elapsed 70m 38s (remain 37m 46s) Loss: 0.4045(0.1033) Grad: 20.5239  LR: 0.00000010  \n",
    "Epoch: [6][52000/78269] Elapsed 72m 2s (remain 36m 23s) Loss: 0.2744(0.1033) Grad: 20.5902  LR: 0.00000010  \n",
    "Epoch: [6][53000/78269] Elapsed 73m 25s (remain 35m 0s) Loss: 0.0804(0.1033) Grad: 9.8693  LR: 0.00000009  \n",
    "Epoch: [6][54000/78269] Elapsed 74m 49s (remain 33m 37s) Loss: 0.0385(0.1033) Grad: 20.7497  LR: 0.00000008  \n",
    "Epoch: [6][55000/78269] Elapsed 76m 12s (remain 32m 14s) Loss: 0.0328(0.1034) Grad: 3.8528  LR: 0.00000007  \n",
    "Epoch: [6][56000/78269] Elapsed 77m 35s (remain 30m 51s) Loss: 0.0012(0.1034) Grad: 1.7573  LR: 0.00000007  \n",
    "Epoch: [6][57000/78269] Elapsed 78m 59s (remain 29m 28s) Loss: 0.0540(0.1034) Grad: 9.7871  LR: 0.00000006  \n",
    "Epoch: [6][58000/78269] Elapsed 80m 24s (remain 28m 5s) Loss: 0.0004(0.1034) Grad: 0.2009  LR: 0.00000006  \n",
    "Epoch: [6][59000/78269] Elapsed 81m 49s (remain 26m 43s) Loss: 0.0012(0.1033) Grad: 0.3285  LR: 0.00000005  \n",
    "Epoch: [6][60000/78269] Elapsed 83m 12s (remain 25m 20s) Loss: 0.0840(0.1035) Grad: 39.1272  LR: 0.00000005  \n",
    "Epoch: [6][61000/78269] Elapsed 84m 35s (remain 23m 56s) Loss: 0.4088(0.1035) Grad: 37.1337  LR: 0.00000004  \n",
    "Epoch: [6][62000/78269] Elapsed 85m 59s (remain 22m 33s) Loss: 0.2186(0.1035) Grad: 2.6124  LR: 0.00000004  \n",
    "Epoch: [6][63000/78269] Elapsed 87m 22s (remain 21m 10s) Loss: 0.0173(0.1038) Grad: 13.1934  LR: 0.00000003  \n",
    "Epoch: [6][64000/78269] Elapsed 88m 45s (remain 19m 47s) Loss: 0.1634(0.1037) Grad: 6.8625  LR: 0.00000003  \n",
    "Epoch: [6][65000/78269] Elapsed 90m 7s (remain 18m 23s) Loss: 0.0138(0.1038) Grad: 7.4357  LR: 0.00000002  \n",
    "Epoch: [6][66000/78269] Elapsed 91m 30s (remain 17m 0s) Loss: 0.0003(0.1039) Grad: 0.0221  LR: 0.00000002  \n",
    "Epoch: [6][67000/78269] Elapsed 92m 52s (remain 15m 37s) Loss: 0.0002(0.1040) Grad: 0.1851  LR: 0.00000002  \n",
    "Epoch: [6][68000/78269] Elapsed 94m 14s (remain 14m 13s) Loss: 0.1367(0.1041) Grad: 31.7022  LR: 0.00000001  \n",
    "Epoch: [6][69000/78269] Elapsed 95m 36s (remain 12m 50s) Loss: 0.0089(0.1040) Grad: 4.0353  LR: 0.00000001  \n",
    "Epoch: [6][70000/78269] Elapsed 96m 58s (remain 11m 27s) Loss: 0.0014(0.1040) Grad: 0.4575  LR: 0.00000001  \n",
    "Epoch: [6][71000/78269] Elapsed 98m 20s (remain 10m 3s) Loss: 0.0044(0.1041) Grad: 0.5259  LR: 0.00000001  \n",
    "Epoch: [6][72000/78269] Elapsed 99m 42s (remain 8m 40s) Loss: 0.0005(0.1041) Grad: 0.3099  LR: 0.00000001  \n",
    "Epoch: [6][73000/78269] Elapsed 101m 4s (remain 7m 17s) Loss: 0.1738(0.1041) Grad: 38.6550  LR: 0.00000000  \n",
    "Epoch: [6][74000/78269] Elapsed 102m 26s (remain 5m 54s) Loss: 0.0055(0.1041) Grad: 1.9344  LR: 0.00000000  \n",
    "Epoch: [6][75000/78269] Elapsed 103m 49s (remain 4m 31s) Loss: 0.4133(0.1041) Grad: 19.1720  LR: 0.00000000  \n",
    "Epoch: [6][76000/78269] Elapsed 105m 11s (remain 3m 8s) Loss: 0.3042(0.1042) Grad: 12.8148  LR: 0.00000000  \n",
    "Epoch: [6][77000/78269] Elapsed 106m 33s (remain 1m 45s) Loss: 0.0042(0.1042) Grad: 0.9930  LR: 0.00000000  \n",
    "Epoch: [6][78000/78269] Elapsed 107m 56s (remain 0m 22s) Loss: 0.2722(0.1043) Grad: 15.4058  LR: 0.00000000  \n",
    "Epoch: [6][78268/78269] Elapsed 108m 19s (remain 0m 0s) Loss: 0.0002(0.1043) Grad: 0.0330  LR: 0.00000000  \n",
    "EVAL: [0/19225] Elapsed 0m 1s (remain 369m 34s) Loss: 0.0001(0.0001) \n",
    "EVAL: [1000/19225] Elapsed 0m 14s (remain 4m 16s) Loss: 1.3322(0.3857) \n",
    "EVAL: [2000/19225] Elapsed 0m 25s (remain 3m 42s) Loss: 0.0000(0.3232) \n",
    "EVAL: [3000/19225] Elapsed 0m 36s (remain 3m 16s) Loss: 0.0000(0.2577) \n",
    "EVAL: [4000/19225] Elapsed 0m 49s (remain 3m 8s) Loss: 0.1669(0.2453) \n",
    "EVAL: [5000/19225] Elapsed 1m 3s (remain 2m 59s) Loss: 0.0001(0.2669) \n",
    "EVAL: [6000/19225] Elapsed 1m 17s (remain 2m 50s) Loss: 0.6223(0.2772) \n",
    "EVAL: [7000/19225] Elapsed 1m 32s (remain 2m 41s) Loss: 0.0033(0.2871) \n",
    "EVAL: [8000/19225] Elapsed 1m 47s (remain 2m 30s) Loss: 0.0020(0.2984) \n",
    "EVAL: [9000/19225] Elapsed 2m 2s (remain 2m 18s) Loss: 0.0000(0.3099) \n",
    "EVAL: [10000/19225] Elapsed 2m 17s (remain 2m 6s) Loss: 0.0079(0.3150) \n",
    "EVAL: [11000/19225] Elapsed 2m 33s (remain 1m 54s) Loss: 0.3436(0.3179) \n",
    "EVAL: [12000/19225] Elapsed 2m 49s (remain 1m 42s) Loss: 0.0000(0.3221) \n",
    "EVAL: [13000/19225] Elapsed 3m 6s (remain 1m 29s) Loss: 0.0001(0.3255) \n",
    "EVAL: [14000/19225] Elapsed 3m 23s (remain 1m 15s) Loss: 0.0000(0.3243) \n",
    "EVAL: [15000/19225] Elapsed 3m 40s (remain 1m 2s) Loss: 0.0003(0.3262) \n",
    "EVAL: [16000/19225] Elapsed 3m 58s (remain 0m 48s) Loss: 0.9963(0.3269) \n",
    "EVAL: [17000/19225] Elapsed 4m 16s (remain 0m 33s) Loss: 0.0001(0.3257) \n",
    "EVAL: [18000/19225] Elapsed 4m 36s (remain 0m 18s) Loss: 0.0008(0.3240) \n",
    "EVAL: [19000/19225] Elapsed 4m 58s (remain 0m 3s) Loss: 2.0782(0.3238) \n",
    "EVAL: [19224/19225] Elapsed 5m 3s (remain 0m 0s) Loss: 0.0063(0.3242) \n",
    "Epoch 6 - avg_train_loss: 0.1043  avg_val_loss: 0.3242  time: 7074s\n",
    "Epoch 6 - Score: 0.4796 - Threshold: 0.01000\n",
    "Our CV score is 0.4893 using a threshold of 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}