Visual_Geolocalization_2.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "h_d595Nlu-U1"
      },
      "source": [
        "# Libs / initial script"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "960P2vRFSOBb",
        "outputId": "2260b55c-3ba3-4e5c-c68e-eb1f3ce4d0ad"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "import shutil\n",
        "\n",
        "#all drive\n",
        "from google.colab import files\n",
        "\n",
        "#mounting drive \n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "#from folder `drive` to outside \n",
        "!cp -r /content/drive/MyDrive/code/* /content"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vWwFrIxf04tx",
        "outputId": "71489d62-2f90-496c-8afb-b7bd914f3a4e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting shap\n",
            "  Downloading shap-0.41.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (572 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m572.6/572.6 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from shap) (1.22.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from shap) (1.10.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from shap) (1.2.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from shap) (1.5.3)\n",
            "Requirement already satisfied: tqdm>4.25.0 in /usr/local/lib/python3.10/dist-packages (from shap) (4.65.0)\n",
            "Requirement already satisfied: packaging>20.9 in /usr/local/lib/python3.10/dist-packages (from shap) (23.1)\n",
            "Collecting slicer==0.0.7 (from shap)\n",
            "  Downloading slicer-0.0.7-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from shap) (0.56.4)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.10/dist-packages (from shap) (2.2.1)\n",
            "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->shap) (0.39.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from numba->shap) (67.7.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->shap) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->shap) (2022.7.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->shap) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->shap) (3.1.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->shap) (1.16.0)\n",
            "Installing collected packages: slicer, shap\n",
            "Successfully installed shap-0.41.0 slicer-0.0.7\n"
          ]
        }
      ],
      "source": [
        "#install dependencies\n",
        "!pip3 install -r requirements.txt > /dev/null\n",
        "!pip3 install shap #verificare"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1cLScNkhxFFf"
      },
      "outputs": [],
      "source": [
        "#unzip folder\n",
        "#if you need to test, uncomment the first and last line. sf-xs is used to train\n",
        "!unzip /content/datasets/tokyo-xs.zip > /dev/null\n",
        "!unzip /content/datasets/sf-xs-001.zip > /dev/null\n",
        "!unzip /content/logs.zip > /dev/null\n",
        "\n",
        "#database for tokyo night\n",
        "!cp -r /content/tokyo_xs/test/database /content/tokyo-night/test/\n",
        "\n",
        "# rename queries of tokyo_xs in queries_v1\n",
        "\n",
        "import os\n",
        "\n",
        "old_name = r\"/content/tokyo_xs/test/queries\"\n",
        "new_name = r\"/content/tokyo_xs/test/queries_v1\"\n",
        "os.rename(old_name, new_name)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "oOjUMtVA-ElR"
      },
      "source": [
        "# Cosface"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "3_TcfYG2yfZo"
      },
      "source": [
        "## Training with CosFace on sf-xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "03ngkbzi9RFZ"
      },
      "source": [
        "Results 03/02/2022: R@1: 52.3, R@5: 66.3, R@10: 71.8, R@20: 76.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FL-36X_7CbY1",
        "outputId": "a87b0aef-f4f0-4fb0-bd9c-f1f8b91c9a08"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2022-12-14 22:18:55   train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3\n",
            "2022-12-14 22:18:55   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='resnet18', batch_size=32, brightness=0.7, classifiers_lr=0.01, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=3, fc_output_dim=512, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10000, lr=1e-05, min_images_per_class=10, num_workers=8, positive_dist_threshold=25, random_resized_crop=0.5, resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val')\n",
            "2022-12-14 22:18:55   The outputs are being saved in logs/default/2022-12-14_22-18-55\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:00<00:00, 134MB/s]\n",
            "2022-12-14 22:18:56   Train only layer3 and layer4 of the resnet18, freeze the previous ones\n",
            "2022-12-14 22:18:57   There are 1 GPUs and 2 CPUs.\n",
            "2022-12-14 22:19:01   Cached dataset cache/small_M10_N5_mipc10.torch does not exist, I'll create it now.\n",
            "2022-12-14 22:19:01   Searching training images in /content/small/train\n",
            "2022-12-14 22:19:02   Found 59650 images\n",
            "2022-12-14 22:19:02   For each image, get its UTM east, UTM north and heading from its path\n",
            "2022-12-14 22:19:02   For each image, get class and group to which it belongs\n",
            "2022-12-14 22:19:02   Group together images belonging to the same class\n",
            "2022-12-14 22:19:02   Group together classes belonging to the same group\n",
            "2022-12-14 22:19:02   Using 1 groups\n",
            "2022-12-14 22:19:02   The 1 groups have respectively the following number of classes [5965]\n",
            "2022-12-14 22:19:02   The 1 groups have respectively the following number of images [59650]\n",
            "2022-12-14 22:19:03   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2022-12-14 22:19:03   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2022-12-14 22:19:03   Start training ...\n",
            "2022-12-14 22:19:03   There are 5965 classes for the first group, each epoch has 10000 iterations with batch_size 32, therefore the model sees each class (on average) 53.6 times per epoch\n",
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████| 10000/10000 [1:13:21<00:00,  2.27it/s]\n",
            "2022-12-14 23:32:25   Epoch 00 in 1:13:21, loss = 7.9503\n",
            "2022-12-14 23:32:25   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [01:06<00:00,  7.57it/s]\n",
            "2022-12-14 23:33:31   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [01:48<00:00, 73.68it/s]\n",
            "2022-12-14 23:35:19   Calculating recalls\n",
            "2022-12-14 23:35:21   Epoch 00 in 1:16:18, < val - #q: 7993; #db: 8015 >: R@1: 78.7, R@5: 88.0\n",
            "100%|███████████████████████████████████████████████████████| 10000/10000 [1:13:19<00:00,  2.27it/s]\n",
            "2022-12-15 00:48:41   Epoch 01 in 1:13:20, loss = 3.3677\n",
            "2022-12-15 00:48:41   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [01:03<00:00,  7.84it/s]\n",
            "2022-12-15 00:49:45   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [01:48<00:00, 73.69it/s]\n",
            "2022-12-15 00:51:34   Calculating recalls\n",
            "2022-12-15 00:51:35   Epoch 01 in 1:16:13, < val - #q: 7993; #db: 8015 >: R@1: 81.9, R@5: 90.0\n",
            "100%|███████████████████████████████████████████████████████| 10000/10000 [1:13:13<00:00,  2.28it/s]\n",
            "2022-12-15 02:04:50   Epoch 02 in 1:13:14, loss = 2.4285\n",
            "2022-12-15 02:04:50   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [01:03<00:00,  7.94it/s]\n",
            "2022-12-15 02:05:53   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [01:47<00:00, 74.54it/s]\n",
            "2022-12-15 02:07:41   Calculating recalls\n",
            "2022-12-15 02:07:42   Epoch 02 in 1:16:06, < val - #q: 7993; #db: 8015 >: R@1: 83.1, R@5: 90.5\n",
            "2022-12-15 02:07:42   Trained for 03 epochs, in total in 3:48:47\n",
            "2022-12-15 02:07:43   Now testing on the test set: < test - #q: 1000; #db: 27191 >\n",
            "2022-12-15 02:07:43   Extracting database descriptors for evaluation/testing\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [03:37<00:00,  7.82it/s]\n",
            "2022-12-15 02:11:20   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:15<00:00, 64.09it/s]\n",
            "2022-12-15 02:11:36   Calculating recalls\n",
            "2022-12-15 02:11:36   < test - #q: 1000; #db: 27191 >: R@1: 52.3, R@5: 66.3, R@10: 71.8, R@20: 76.3\n",
            "2022-12-15 02:11:36   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --loss_function cosface"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "lqv3eUJPyt40"
      },
      "source": [
        "## Evaluation on tokyo_xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "_ZR93Du_9D5B"
      },
      "source": [
        "Results 03/02/2033: R@1: 69.5, R@5: 84.8, R@10: 89.2, R@20: 92.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "57lHGHP8S9wH",
        "outputId": "f14da4d7-f32e-4df3-c1f2-b2340ab85070"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:27:26   eval.py --dataset_folder /content/tokyo_xs/ --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "2023-04-21 08:27:26   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_cosface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 08:27:26   The outputs are being saved in logs/default/2023-04-21_08-27-26\n",
            "2023-04-21 08:27:26   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:27:26   Loading model from /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:19<00:00,  5.74it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:05<00:00, 60.12it/s]\n",
            "2023-04-21 08:29:52   < test - #q: 315; #db: 12771 >: R@1: 69.5, R@5: 84.8, R@10: 89.2, R@20: 92.7\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo_xs/  --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m8XMVhfXFF2B"
      },
      "outputs": [],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo_xs/  --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "6KjS8FDRy5hB"
      },
      "source": [
        "## Evaluation on sf-xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "zi2_QM_t9Wqw"
      },
      "source": [
        "Results 03/02/2022: R@1: 52.3, R@5: 66.3, R@10: 71.8, R@20: 76.3"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "zoaDvgxK9Jc-"
      },
      "source": [
        "Results "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7sM8w4R-O48-",
        "outputId": "423c2734-7ec6-46cc-b374-b42af449650b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 09:13:27   eval.py --dataset_folder /content/small/ --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "2023-04-21 09:13:27   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_cosface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-21 09:13:27   The outputs are being saved in logs/default/2023-04-21_09-13-27\n",
            "2023-04-21 09:13:27   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 09:13:27   Loading model from /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [04:42<00:00,  6.02it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:16<00:00, 61.98it/s]\n",
            "2023-04-21 09:18:28   < test - #q: 1000; #db: 27191 >: R@1: 47.0, R@5: 63.1, R@10: 68.4, R@20: 73.8\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/small/  --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "w2lx7quay_6T"
      },
      "source": [
        "## Evaluation on tokyo-night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "NsqfcZhS9Zax"
      },
      "source": [
        "Results 03/02/2023: R@1: 49.5, R@5: 73.3, R@10: 80.0, R@20: 84.8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HSKteSXGRQKF"
      },
      "outputs": [],
      "source": [
        "#use database of tokyoxs\n",
        "!cp -r /content/tokyo_xs/test/database /content/tokyo-night"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "14F7KmCHT3C9",
        "outputId": "17750fc8-c0b9-43c6-9fc9-94b7e78eb9f8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:35:20   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "2023-04-21 08:35:20   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_cosface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 08:35:20   The outputs are being saved in logs/default/2023-04-21_08-35-20\n",
            "2023-04-21 08:35:20   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:35:20   Loading model from /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:04<00:00,  6.44it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 33.80it/s]\n",
            "2023-04-21 08:37:30   < test - #q: 105; #db: 12771 >: R@1: 49.5, R@5: 73.3, R@10: 80.0, R@20: 84.8\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "A8t_S40jORxm"
      },
      "source": [
        "# Trying different loss function\n",
        "\n",
        "\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "-8JwZjo1-TEh"
      },
      "source": [
        "# Arcface"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "FQvJDmdJ1pj8"
      },
      "source": [
        "## Training sf-xs with Arc Face"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "9_-rwI_j99G6"
      },
      "source": [
        "Results 03/02/2023: R@1: 51.9, R@5: 66.6, R@10: 71.3, R@20: 75.5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lmC8zffKOakD",
        "outputId": "6fa9ba8e-6585-4d5b-8f38-b183e6b6eac8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-01-22 08:38:17   train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --loss_function arcface\n",
            "2023-01-22 08:38:17   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='ResNet18', batch_size=32, brightness=0.7, classifiers_lr=0.01, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=3, fc_output_dim=512, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10000, loss_function='arcface', lr=1e-05, min_images_per_class=10, num_workers=8, positive_dist_threshold=25, random_resized_crop=0.5, resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val')\n",
            "2023-01-22 08:38:17   The outputs are being saved in logs/default/2023-01-22_08-38-17\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:00<00:00, 94.8MB/s]\n",
            "2023-01-22 08:38:18   Train only layer3 and layer4 of the ResNet18, freeze the previous ones\n",
            "2023-01-22 08:38:18   There are 1 GPUs and 12 CPUs.\n",
            "2023-01-22 08:38:23   Cached dataset cache/small_M10_N5_mipc10.torch does not exist, I'll create it now.\n",
            "2023-01-22 08:38:23   Searching training images in /content/small/train\n",
            "2023-01-22 08:38:23   Found 59650 images\n",
            "2023-01-22 08:38:23   For each image, get its UTM east, UTM north and heading from its path\n",
            "2023-01-22 08:38:23   For each image, get class and group to which it belongs\n",
            "2023-01-22 08:38:23   Group together images belonging to the same class\n",
            "2023-01-22 08:38:23   Group together classes belonging to the same group\n",
            "2023-01-22 08:38:23   Using arcface loss function.\n",
            "2023-01-22 08:38:23   Using 1 groups\n",
            "2023-01-22 08:38:23   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-01-22 08:38:23   The 1 groups have respectively the following number of images [59650]\n",
            "2023-01-22 08:38:24   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-01-22 08:38:24   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-01-22 08:38:24   Start training ...\n",
            "2023-01-22 08:38:24   There are 5965 classes for the first group, each epoch has 10000 iterations with batch_size 32, therefore the model sees each class (on average) 53.6 times per epoch\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [16:52<00:00,  9.88it/s]\n",
            "2023-01-22 08:55:16   Epoch 00 in 0:16:52, loss = 17.5596\n",
            "2023-01-22 08:55:16   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:12<00:00, 38.75it/s]\n",
            "2023-01-22 08:55:29   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|██████████████████████████████████████████████████████████| 7993/7993 [00:52<00:00, 153.68it/s]\n",
            "2023-01-22 08:56:21   Calculating recalls\n",
            "2023-01-22 08:56:23   Epoch 00 in 0:17:58, < val - #q: 7993; #db: 8015 >: R@1: 78.2, R@5: 87.9\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [16:46<00:00,  9.94it/s]\n",
            "2023-01-22 09:13:10   Epoch 01 in 0:16:46, loss = 7.2078\n",
            "2023-01-22 09:13:10   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:12<00:00, 39.30it/s]\n",
            "2023-01-22 09:13:23   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|██████████████████████████████████████████████████████████| 7993/7993 [00:52<00:00, 153.35it/s]\n",
            "2023-01-22 09:14:15   Calculating recalls\n",
            "2023-01-22 09:14:16   Epoch 01 in 0:17:53, < val - #q: 7993; #db: 8015 >: R@1: 81.4, R@5: 89.7\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [16:46<00:00,  9.93it/s]\n",
            "2023-01-22 09:31:04   Epoch 02 in 0:16:46, loss = 4.8806\n",
            "2023-01-22 09:31:04   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:12<00:00, 40.06it/s]\n",
            "2023-01-22 09:31:16   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|██████████████████████████████████████████████████████████| 7993/7993 [00:51<00:00, 154.82it/s]\n",
            "2023-01-22 09:32:08   Calculating recalls\n",
            "2023-01-22 09:32:09   Epoch 02 in 0:17:52, < val - #q: 7993; #db: 8015 >: R@1: 82.8, R@5: 90.5\n",
            "2023-01-22 09:32:10   Trained for 03 epochs, in total in 0:53:52\n",
            "2023-01-22 09:32:10   Now testing on the test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-01-22 09:32:10   Extracting database descriptors for evaluation/testing\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [00:42<00:00, 39.99it/s]\n",
            "2023-01-22 09:32:52   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|██████████████████████████████████████████████████████████| 1000/1000 [00:07<00:00, 133.50it/s]\n",
            "2023-01-22 09:33:00   Calculating recalls\n",
            "2023-01-22 09:33:00   < test - #q: 1000; #db: 27191 >: R@1: 51.9, R@5: 66.6, R@10: 71.3, R@20: 75.5\n",
            "2023-01-22 09:33:00   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --loss_function arcface"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "LG4U8hFKp8Hx"
      },
      "source": [
        "## Evaluation on tokyo_xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "KoOTUhQU-Dy3"
      },
      "source": [
        "Results 03/02/2022: R@1: 67.3, R@5: 80.6, R@10: 86.0, R@20: 92.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OwVXJkysqHrz",
        "outputId": "71091007-3e59-4626-80b6-a21af9e5ddda"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:38:53   eval.py --dataset_folder /content/tokyo_xs/ --resume_model /content/logs/content/logs/default/trained_with_arcface/best_model.pth\n",
            "2023-04-21 08:38:53   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_arcface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 08:38:53   The outputs are being saved in logs/default/2023-04-21_08-38-53\n",
            "2023-04-21 08:38:53   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:38:53   Loading model from /content/logs/content/logs/default/trained_with_arcface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:04<00:00,  6.44it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:07<00:00, 44.44it/s]\n",
            "2023-04-21 08:41:07   < test - #q: 315; #db: 12771 >: R@1: 67.3, R@5: 80.6, R@10: 86.0, R@20: 92.1\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo_xs/  --resume_model /content/logs/content/logs/default/trained_with_arcface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "f5123uAdoGOA"
      },
      "source": [
        "## Evaluation on sf-xs(test)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "AGe1znZI-HxG"
      },
      "source": [
        "Results 03/02/2023: R@1: 51.9, R@5: 66.6, R@10: 71.3, R@20: 75.5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9uZCxmT6oEAA",
        "outputId": "73e74674-d484-4494-8b4c-eec68f33a4fc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:41:10   eval.py --dataset_folder /content/small/ --resume_model /content/logs/content/logs/default/trained_with_arcface/best_model.pth\n",
            "2023-04-21 08:41:10   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_arcface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-21 08:41:10   The outputs are being saved in logs/default/2023-04-21_08-41-10\n",
            "2023-04-21 08:41:11   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:41:11   Loading model from /content/logs/content/logs/default/trained_with_arcface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [03:41<00:00,  7.66it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:16<00:00, 61.97it/s]\n",
            "2023-04-21 08:45:11   < test - #q: 1000; #db: 27191 >: R@1: 51.9, R@5: 66.6, R@10: 71.3, R@20: 75.5\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/small/  --resume_model /content/logs/content/logs/default/trained_with_arcface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ko7QJtcdq98a"
      },
      "source": [
        "## Evaluation on tokyo-night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "5RpdIzEM-KFI"
      },
      "source": [
        "Results 03/02/2023: R@1: 47.6, R@5: 70.5, R@10: 77.1, R@20: 85.7\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZPA9-6rerSbP",
        "outputId": "2d014100-b93d-4815-e50a-de618030bab7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:45:16   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/trained_with_arcface/best_model.pth\n",
            "2023-04-21 08:45:16   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_arcface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 08:45:16   The outputs are being saved in logs/default/2023-04-21_08-45-16\n",
            "2023-04-21 08:45:16   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:45:16   Loading model from /content/logs/content/logs/default/trained_with_arcface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:03<00:00,  6.46it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 37.69it/s]\n",
            "2023-04-21 08:47:25   < test - #q: 105; #db: 12771 >: R@1: 47.6, R@5: 70.5, R@10: 77.1, R@20: 85.7\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/trained_with_arcface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "XZdaPuS6wNB9"
      },
      "source": [
        "# SphereFace"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "TYeUeG4rwRiw"
      },
      "source": [
        "## Training sf-xs with SphereFace"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "QYFKVIbK-nB1"
      },
      "source": [
        "Results 03/02/2023: R@1: 50.6, R@5: 65.0, R@10: 71.3, R@20: 75.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rZRWqVm-wcXS",
        "outputId": "4a45f0f4-ebaa-48e5-d83d-c3ff7ef1f0f1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-01-23 18:02:58   train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --loss_function sphereface\n",
            "2023-01-23 18:02:58   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='ResNet18', batch_size=32, brightness=0.7, classifiers_lr=0.01, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=3, fc_output_dim=512, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10000, loss_function='sphereface', lr=1e-05, min_images_per_class=10, num_workers=8, positive_dist_threshold=25, random_resized_crop=0.5, resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val')\n",
            "2023-01-23 18:02:58   The outputs are being saved in logs/default/2023-01-23_18-02-58\n",
            "2023-01-23 18:02:58   Train only layer3 and layer4 of the ResNet18, freeze the previous ones\n",
            "2023-01-23 18:02:59   There are 1 GPUs and 12 CPUs.\n",
            "2023-01-23 18:03:01   Using cached dataset cache/small_M10_N5_mipc10.torch\n",
            "2023-01-23 18:03:01   Using sphereface loss function.\n",
            "2023-01-23 18:03:01   Using 1 groups\n",
            "2023-01-23 18:03:01   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-01-23 18:03:01   The 1 groups have respectively the following number of images [59650]\n",
            "2023-01-23 18:03:01   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-01-23 18:03:01   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-01-23 18:03:01   Start training ...\n",
            "2023-01-23 18:03:01   There are 5965 classes for the first group, each epoch has 10000 iterations with batch_size 32, therefore the model sees each class (on average) 53.6 times per epoch\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [17:02<00:00,  9.78it/s]\n",
            "2023-01-23 18:20:04   Epoch 00 in 0:17:02, loss = 8.5344\n",
            "2023-01-23 18:20:04   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:12<00:00, 38.62it/s]\n",
            "2023-01-23 18:20:17   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|██████████████████████████████████████████████████████████| 7993/7993 [00:53<00:00, 149.77it/s]\n",
            "2023-01-23 18:21:10   Calculating recalls\n",
            "2023-01-23 18:21:12   Epoch 00 in 0:18:10, < val - #q: 7993; #db: 8015 >: R@1: 78.6, R@5: 87.8\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [16:56<00:00,  9.84it/s]\n",
            "2023-01-23 18:38:09   Epoch 01 in 0:16:57, loss = 3.0982\n",
            "2023-01-23 18:38:09   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:13<00:00, 37.76it/s]\n",
            "2023-01-23 18:38:22   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|██████████████████████████████████████████████████████████| 7993/7993 [00:54<00:00, 147.93it/s]\n",
            "2023-01-23 18:39:16   Calculating recalls\n",
            "2023-01-23 18:39:18   Epoch 01 in 0:18:05, < val - #q: 7993; #db: 8015 >: R@1: 81.7, R@5: 89.9\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [17:01<00:00,  9.79it/s]\n",
            "2023-01-23 18:56:21   Epoch 02 in 0:17:02, loss = 2.0947\n",
            "2023-01-23 18:56:21   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:12<00:00, 39.25it/s]\n",
            "2023-01-23 18:56:33   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|██████████████████████████████████████████████████████████| 7993/7993 [00:53<00:00, 149.90it/s]\n",
            "2023-01-23 18:57:27   Calculating recalls\n",
            "2023-01-23 18:57:28   Epoch 02 in 0:18:09, < val - #q: 7993; #db: 8015 >: R@1: 82.8, R@5: 90.7\n",
            "2023-01-23 18:57:29   Trained for 03 epochs, in total in 0:54:30\n",
            "2023-01-23 18:57:29   Now testing on the test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-01-23 18:57:29   Extracting database descriptors for evaluation/testing\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [00:42<00:00, 39.73it/s]\n",
            "2023-01-23 18:58:11   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|██████████████████████████████████████████████████████████| 1000/1000 [00:07<00:00, 129.48it/s]\n",
            "2023-01-23 18:58:19   Calculating recalls\n",
            "2023-01-23 18:58:20   < test - #q: 1000; #db: 27191 >: R@1: 50.6, R@5: 65.0, R@10: 71.3, R@20: 75.1\n",
            "2023-01-23 18:58:20   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --loss_function sphereface"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "JES-K4o1wcxO"
      },
      "source": [
        "## Evaluation on tokyo_xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ToJPAOBh-rOr"
      },
      "source": [
        "Results 03/02/2023: R@1: 68.9, R@5: 84.1, R@10: 88.9, R@20: 92.4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AZATo9Q9whIf",
        "outputId": "5132d600-c375-46d9-f9b6-48483cfe6e60"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:48:17   eval.py --dataset_folder /content/tokyo_xs/ --resume_model /content/logs/content/logs/default/trained_with_sphereface/best_model.pth\n",
            "2023-04-21 08:48:17   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_sphereface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 08:48:17   The outputs are being saved in logs/default/2023-04-21_08-48-17\n",
            "2023-04-21 08:48:18   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:48:18   Loading model from /content/logs/content/logs/default/trained_with_sphereface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:03<00:00,  6.49it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:04<00:00, 65.07it/s]\n",
            "2023-04-21 08:50:28   < test - #q: 315; #db: 12771 >: R@1: 68.9, R@5: 84.1, R@10: 88.9, R@20: 92.4\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo_xs/  --resume_model /content/logs/content/logs/default/trained_with_sphereface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "3fwUPdYnwhsr"
      },
      "source": [
        "## Evaluation on sf-xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "bGWq1w5m-uc2"
      },
      "source": [
        "Results 03/02/2023: R@1: 50.5, R@5: 65.0, R@10: 71.3, R@20: 75.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P2E_wuaawkU4",
        "outputId": "a0ead826-0184-411b-b607-84a110978959"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:50:32   eval.py --dataset_folder /content/small/ --resume_model /content/logs/content/logs/default/trained_with_sphereface/best_model.pth\n",
            "2023-04-21 08:50:32   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_sphereface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-21 08:50:32   The outputs are being saved in logs/default/2023-04-21_08-50-32\n",
            "2023-04-21 08:50:33   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:50:33   Loading model from /content/logs/content/logs/default/trained_with_sphereface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [03:40<00:00,  7.70it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:16<00:00, 61.65it/s]\n",
            "2023-04-21 08:54:32   < test - #q: 1000; #db: 27191 >: R@1: 50.5, R@5: 65.0, R@10: 71.3, R@20: 75.1\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/small/  --resume_model /content/logs/content/logs/default/trained_with_sphereface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "lJn8WgTzwkmn"
      },
      "source": [
        "## Evaluation on tokyo-night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "PV1Oqe6V-924"
      },
      "source": [
        "Results 03/02/2023: R@1: 46.7, R@5: 72.4, R@10: 79.0, R@20: 83.8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NpTi5YpZwnw7",
        "outputId": "ad125b40-68bc-43e8-b67b-54967efa6b3b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:54:36   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/trained_with_sphereface/best_model.pth\n",
            "2023-04-21 08:54:36   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_sphereface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 08:54:36   The outputs are being saved in logs/default/2023-04-21_08-54-36\n",
            "2023-04-21 08:54:36   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:54:36   Loading model from /content/logs/content/logs/default/trained_with_sphereface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:04<00:00,  6.44it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 36.57it/s]\n",
            "2023-04-21 08:56:46   < test - #q: 105; #db: 12771 >: R@1: 46.7, R@5: 72.4, R@10: 79.0, R@20: 83.8\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/trained_with_sphereface/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "qmXtFvmyWCgT"
      },
      "source": [
        "# DA ELMIINARE Laplace Approximation"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Bh_enx2hhX45"
      },
      "source": [
        "Si può cancellare?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Nwz5eH4WHE3",
        "outputId": "01e84695-81fa-4a3a-f6dc-e35bccee1103"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-01-30 18:19:11   train.py --dataset_folder /content/small --groups_num 1 --iterations_per_epoch 10 --epochs_num 1 --loss_function cosface --enable_laplace True\n",
            "2023-01-30 18:19:11   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='ResNet18', batch_size=32, brightness=0.7, classifiers_lr=0.01, contrast=0.7, dataset_folder='/content/small', device='cuda', enable_laplace=True, epochs_num=1, fc_output_dim=512, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10, loss_function='cosface', lr=1e-05, min_images_per_class=10, num_workers=8, positive_dist_threshold=25, random_resized_crop=0.5, resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val')\n",
            "2023-01-30 18:19:11   The outputs are being saved in logs/default/2023-01-30_18-19-11\n",
            "2023-01-30 18:19:11   Train only layer3 and layer4 of the ResNet18, freeze the previous ones\n",
            "2023-01-30 18:19:12   There are 1 GPUs and 2 CPUs.\n",
            "2023-01-30 18:19:15   Using cached dataset cache/small_M10_N5_mipc10.torch\n",
            "2023-01-30 18:19:15   Using cosface loss function.\n",
            "2023-01-30 18:19:15   Using 1 groups\n",
            "2023-01-30 18:19:15   The 1 groups have respectively the following number of classes [740]\n",
            "2023-01-30 18:19:16   The 1 groups have respectively the following number of images [7400]\n",
            "2023-01-30 18:19:16   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-01-30 18:19:16   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-01-30 18:19:16   Start training ...\n",
            "2023-01-30 18:19:16   There are 740 classes for the first group, each epoch has 10 iterations with batch_size 32, therefore the model sees each class (on average) 0.4 times per epoch\n",
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:474: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████████| 10/10 [00:08<00:00,  1.13it/s]\n",
            "2023-01-30 18:19:26   Epoch 00 in 0:00:09, loss = 30.5192\n",
            "Computing the Laplace Approximation for Bayesian hypothesis generation...\n",
            "2023-01-30 18:19:26   \n",
            "Traceback (most recent call last):\n",
            "  File \"train.py\", line 182, in <module>\n",
            "    compute_laplace(args, model, dataloader)\n",
            "  File \"/content/laplace.py\", line 28, in compute_laplace\n",
            "    hessians = get_hessian(args, model, train_loader)\n",
            "  File \"/content/laplace.py\", line 39, in get_hessian\n",
            "    m, n = W.shape\n",
            "ValueError: not enough values to unpack (expected 2, got 1)\n",
            "\n",
            "2023-01-30 18:19:26   Experiment finished (with some errors)\n"
          ]
        }
      ],
      "source": [
        "!python train.py --dataset_folder /content/small --groups_num 1 --iterations_per_epoch 10 --epochs_num 1 --loss_function cosface --enable_laplace True"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "hSmE5kBLd6FK"
      },
      "source": [
        "# Step 4: GRL TRAIN"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "W0XbpyxJ_LGw"
      },
      "source": [
        "Results 03/02/2023: R@1: 54.7, R@5: 68.1, R@10: 72.8, R@20: 76.4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d5nGJpcgd7z6",
        "outputId": "3f10d3a1-6929-4075-d480-5bfd4a8def24"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "usage: train.py\n",
            "       [-h]\n",
            "       [--wd WD]\n",
            "       [--M M]\n",
            "       [--alpha ALPHA]\n",
            "       [--N N]\n",
            "       [--L L]\n",
            "       [--groups_num GROUPS_NUM]\n",
            "       [--min_images_per_class MIN_IMAGES_PER_CLASS]\n",
            "       [--backbone {VGG16,ResNet18,ResNet50,ResNet101,ResNet152,convnext_tiny,efficientnet_v2_s}]\n",
            "       [--fc_output_dim FC_OUTPUT_DIM]\n",
            "       [--loss_function LOSS_FUNCTION]\n",
            "       [--use_amp16]\n",
            "       [--augmentation_device {cuda,cpu}]\n",
            "       [--batch_size BATCH_SIZE]\n",
            "       [--epochs_num EPOCHS_NUM]\n",
            "       [--iterations_per_epoch ITERATIONS_PER_EPOCH]\n",
            "       [--lr LR]\n",
            "       [--classifiers_lr CLASSIFIERS_LR]\n",
            "       [--brightness BRIGHTNESS]\n",
            "       [--contrast CONTRAST]\n",
            "       [--hue HUE]\n",
            "       [--saturation SATURATION]\n",
            "       [--random_resized_crop RANDOM_RESIZED_CROP]\n",
            "       [--infer_batch_size INFER_BATCH_SIZE]\n",
            "       [--positive_dist_threshold POSITIVE_DIST_THRESHOLD]\n",
            "       [--resume_train RESUME_TRAIN]\n",
            "       [--resume_model RESUME_MODEL]\n",
            "       [--device {cuda,cpu}]\n",
            "       [--seed SEED]\n",
            "       [--num_workers NUM_WORKERS]\n",
            "       [--dataset_folder DATASET_FOLDER]\n",
            "       [--save_dir SAVE_DIR]\n",
            "       [--k K]\n",
            "       [--ss_w SS_W]\n",
            "       [--consistency_w CONSISTENCY_W]\n",
            "       [--features_wise_w FEATURES_WISE_W]\n",
            "       [--qp_threshold QP_THRESHOLD]\n",
            "       [--num_reranked_preds NUM_RERANKED_PREDS]\n",
            "       [--kernel_sizes KERNEL_SIZES [KERNEL_SIZES ...]]\n",
            "       [--channels CHANNELS [CHANNELS ...]]\n",
            "       [--multi_scale]\n",
            "       [--select_resolutions SELECT_RESOLUTIONS [SELECT_RESOLUTIONS ...]]\n",
            "       [--multi_scale_method {avg,sum,max,min}]\n",
            "       [--grl_param GRL_PARAM]\n",
            "       [--night_test NIGHT_TEST]\n",
            "       [--night_brightness NIGHT_BRIGHTNESS]\n",
            "       [--source_dir SOURCE_DIR]\n",
            "       [--target_dir TARGET_DIR]\n",
            "       [--test_method TEST_METHOD]\n",
            "       [--optim {adam,sgd}]\n",
            "       [--resize RESIZE RESIZE]\n",
            "train.py: error: argument --epochs_num: invalid int value: '3--iterations_per_epoch'\n",
            "^C\n"
          ]
        }
      ],
      "source": [
        "!python train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3--iterations_per_epoch 10000 --loss_function cosface --grl_param 0.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RSgnPOGAwmHI",
        "outputId": "10080902-aba3-4660-f8ad-49fb193b675c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-02-05 08:49:57   train.py --dataset_folder /content/small --groups_num 1 --epochs_num 1 --iterations_per_epoch 10 --loss_function cosface --grl_param 0.3 --source_dir /content/small/ --target_dir /content/night_target/\n",
            "2023-02-05 08:49:57   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='ResNet18', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=1, fc_output_dim=512, features_wise_w=10, grl_param=0.3, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method='avg', night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=8, optim='adam', positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resize=[480, 640], resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], source_dir='/content/small/', ss_w=1, target_dir='/content/night_target/', test_method='hard_resize', test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val', wd=None)\n",
            "2023-02-05 08:49:57   The outputs are being saved in logs/default/2023-02-05_08-49-57\n",
            "2023-02-05 08:49:57   Gradient Reversal Layer is enabled with parameter = 0.3\n",
            "2023-02-05 08:49:57   Train only layer3 and layer4 of the ResNet18, freeze the previous ones\n",
            "2023-02-05 08:49:57   There are 1 GPUs and 2 CPUs.\n",
            "2023-02-05 08:49:59   Using cached dataset cache/small_M10_N5_mipc10.torch\n",
            "2023-02-05 08:49:59   GrlDataset has 3 domain classes\n",
            "2023-02-05 08:49:59   Using cosface loss function.\n",
            "2023-02-05 08:49:59   Using 1 groups\n",
            "2023-02-05 08:49:59   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-02-05 08:49:59   The 1 groups have respectively the following number of images [59650]\n",
            "2023-02-05 08:50:00   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-02-05 08:50:00   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-05 08:50:00   Start training ...\n",
            "2023-02-05 08:50:00   There are 5965 classes for the first group, each epoch has 10 iterations with batch_size 32, therefore the model sees each class (on average) 0.1 times per epoch\n",
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████████| 10/10 [00:26<00:00,  2.68s/it]\n",
            "2023-02-05 08:50:28   Epoch 00 in 0:00:28, loss = 38.1683\n",
            "2023-02-05 08:50:28   Average GRL epoch loss (* alpha = 0.3): 0.2073\n",
            "2023-02-05 08:50:28   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [01:08<00:00,  7.34it/s]\n",
            "2023-02-05 08:51:36   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [02:00<00:00, 66.21it/s]\n",
            "2023-02-05 08:53:37   Calculating recalls\n",
            "2023-02-05 08:53:40   Epoch 00 in 0:03:39, < val - #q: 7993; #db: 8015 >: R@1: 30.2, R@5: 43.6\n",
            "2023-02-05 08:53:40   Trained for 01 epochs, in total in 0:03:43\n",
            "2023-02-05 08:53:40   Now testing on the test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-05 08:53:40   Extracting database descriptors for evaluation/testing\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [03:44<00:00,  7.56it/s]\n",
            "2023-02-05 08:57:25   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:17<00:00, 56.99it/s]\n",
            "2023-02-05 08:57:43   Calculating recalls\n"
          ]
        }
      ],
      "source": [
        "# PROVA\n",
        "!python train.py --dataset_folder /content/small --groups_num 1 --epochs_num 1 --iterations_per_epoch 10 --loss_function cosface --grl_param 0.3 --source_dir /content/small/ --target_dir /content/night_target/"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "-JbVU73BlM14"
      },
      "source": [
        "# Experiment on tokyo night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "w445dpM0_RKh"
      },
      "source": [
        "Results 03/02/2023 R@1: 60.0, R@5: 78.1, R@10: 81.9, R@20: 87.6"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "20NRa4NVyBbd",
        "outputId": "7fedbdfb-c4bf-4484-8e74-1367c9086d69"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 08:59:47   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True\n",
            "2023-04-21 08:59:47   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 08:59:47   The outputs are being saved in logs/default/2023-04-21_08-59-47\n",
            "2023-04-21 08:59:48   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 08:59:48   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:33<00:00,  5.20it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 51.97it/s]\n",
            "2023-04-21 09:02:25   < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 78.1, R@10: 81.9, R@20: 87.6\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "6pjaK7QClRFH"
      },
      "source": [
        "# GRL TRAIN (AGAIN)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bvR06VxflTPS",
        "outputId": "94243617-5298-4dcd-d78a-5b2256e9f9d0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-02-03 15:22:53   train.py --dataset_folder /content/small --source_dir /content/small/train --target /content/night_target --groups_num 1 --epochs_num 1 --iterations_per_epoch 10 --loss_function cosface --grl_param 0.3\n",
            "2023-02-03 15:22:53   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='ResNet18', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=1, fc_output_dim=512, features_wise_w=10, grl_param=0.3, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method=None, night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=8, positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[1, 2, 5, 10], source_dir='/content/small/train', ss_w=1, target_dir='/content/night_target', test_method='hard_resize', test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val')\n",
            "2023-02-03 15:22:53   The outputs are being saved in logs/default/2023-02-03_15-22-53\n",
            "2023-02-03 15:22:53   Gradient Reversal Layer is enabled with parameter = 0.3\n",
            "2023-02-03 15:22:53   Train only layer3 and layer4 of the ResNet18, freeze the previous ones\n",
            "2023-02-03 15:22:54   There are 1 GPUs and 2 CPUs.\n",
            "2023-02-03 15:22:58   Using cached dataset cache/small_M10_N5_mipc10.torch\n",
            "2023-02-03 15:22:58   GrlDataset has 3 domain classes\n",
            "2023-02-03 15:22:58   Using cosface loss function.\n",
            "2023-02-03 15:22:58   Using 1 groups\n",
            "2023-02-03 15:22:58   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-02-03 15:22:58   The 1 groups have respectively the following number of images [59650]\n",
            "2023-02-03 15:22:59   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-02-03 15:22:59   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-03 15:22:59   Start training ...\n",
            "2023-02-03 15:22:59   There are 5965 classes for the first group, each epoch has 10 iterations with batch_size 32, therefore the model sees each class (on average) 0.1 times per epoch\n",
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████████| 10/10 [00:20<00:00,  2.09s/it]\n",
            "2023-02-03 15:23:23   Epoch 00 in 0:00:24, loss = 38.1682\n",
            "2023-02-03 15:23:23   Average GRL epoch loss (* alpha = 0.3): 0.2080\n",
            "2023-02-03 15:23:23   Extracting database descriptors for evaluation/testing\n",
            " 64%|███████████████████████████████████████▎                     | 323/501 [00:40<00:22,  7.89it/s]\n",
            "2023-02-03 15:24:04   \n",
            "Traceback (most recent call last):\n",
            "  File \"train.py\", line 217, in <module>\n",
            "    recalls, recalls_str = test.test(args, val_ds, model)\n",
            "  File \"/content/test.py\", line 32, in test\n",
            "    descriptors = descriptors.cpu().numpy()\n",
            "KeyboardInterrupt\n",
            "\n",
            "2023-02-03 15:24:04   Experiment finished (with some errors)\n",
            "^C\n"
          ]
        }
      ],
      "source": [
        "## ????\n",
        "!python train.py --dataset_folder /content/small --source_dir /content/small/train --target /content/night_target --groups_num 1 --epochs_num 1 --iterations_per_epoch 10 --loss_function cosface --grl_param 0.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HoCVmDI14sHJ",
        "outputId": "c33d6351-48e0-4514-aa9f-25085189ac10"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 13:53:27   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True\n",
            "2023-04-21 13:53:27   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 13:53:27   The outputs are being saved in logs/default/2023-04-21_13-53-27\n",
            "2023-04-21 13:53:27   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 13:53:27   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:33<00:00,  5.20it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 50.31it/s]\n",
            "2023-04-21 13:56:05   < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 78.1, R@10: 81.9, R@20: 87.6\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True #file in logs.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UqQOi7PSGi82",
        "outputId": "34aaab42-ba7d-401c-843a-9f4d11f37562"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 13:56:50   eval.py --dataset_folder /content/small/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3\n",
            "2023-04-21 13:56:50   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-21 13:56:50   The outputs are being saved in logs/default/2023-04-21_13-56-50\n",
            "2023-04-21 13:56:50   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 13:56:50   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [03:42<00:00,  7.66it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:15<00:00, 65.21it/s]\n",
            "2023-04-21 14:00:51   < test - #q: 1000; #db: 27191 >: R@1: 54.7, R@5: 68.1, R@10: 72.8, R@20: 76.5\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/small/  --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3  #file in logs.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V5i6p8JlHt1F",
        "outputId": "d00166a6-bf7d-4dbd-8c05-6aee40f1ac56"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 14:01:49   eval.py --dataset_folder /content/tokyo_xs/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3\n",
            "2023-04-21 14:01:49   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 14:01:49   The outputs are being saved in logs/default/2023-04-21_14-01-49\n",
            "2023-04-21 14:01:49   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 14:01:49   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:04<00:00,  6.41it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:06<00:00, 51.78it/s]\n",
            "2023-04-21 14:04:02   < test - #q: 315; #db: 12771 >: R@1: 73.0, R@5: 87.3, R@10: 88.9, R@20: 93.7\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo_xs/  --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3  #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "_OwJblYv499D"
      },
      "source": []
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "nXGhVU2d36wR"
      },
      "source": [
        "#Step 5: Re - ranking"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "72rg84pQ4BuR"
      },
      "source": [
        "## Geowarp"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "vNyNaZlZ4F-0"
      },
      "source": [
        "Inspired by [GMBERTON - Geo_Warp, github](https://github.com/gmberton/geo_warp)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "urb-QK064E7c",
        "outputId": "dbd15b5e-a73e-4277-c810-a5b0e3031e43"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "You are using:  cuda:0\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "#use GPU if available \n",
        "DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\") #'cpu' # 'cuda' or 'cpu'\n",
        "print(\"You are using: \",DEVICE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v-6X4iuq4uT_",
        "outputId": "1e45f25d-7e06-464e-e8fd-dd34a226f89c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'CosPlace'...\n",
            "remote: Enumerating objects: 203, done.\u001b[K\n",
            "remote: Total 203 (delta 0), reused 0 (delta 0), pack-reused 203\u001b[K\n",
            "Receiving objects: 100% (203/203), 51.91 KiB | 916.00 KiB/s, done.\n",
            "Resolving deltas: 100% (112/112), done.\n",
            "Cloning into 'geo_warp'...\n",
            "remote: Enumerating objects: 103, done.\u001b[K\n",
            "remote: Counting objects: 100% (103/103), done.\u001b[K\n",
            "remote: Compressing objects: 100% (86/86), done.\u001b[K\n",
            "remote: Total 103 (delta 46), reused 52 (delta 10), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (103/103), 656.28 KiB | 2.67 MiB/s, done.\n",
            "Resolving deltas: 100% (46/46), done.\n"
          ]
        }
      ],
      "source": [
        "# download code of CosPlace\n",
        "!git clone \"https://github.com/gmberton/CosPlace\" \n",
        "#!rm -r \"/content/CosPlace\"\n",
        "\n",
        "# download code of GeoWarp\n",
        "!git clone \"https://github.com/gmberton/geo_warp\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9LtnnZUQ5E-c",
        "outputId": "4e0df70d-47aa-42b8-e2d4-703b8aafff9a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ok\n"
          ]
        }
      ],
      "source": [
        "#se già runnato prima evitare gli ipmort TRANNE COSPLACE E GEOWARP\n",
        "import os\n",
        "import sys\n",
        "import torch\n",
        "import logging\n",
        "import multiprocessing\n",
        "import numpy as np\n",
        "#import torchvision.transforms -> è in requirements.txt\n",
        "from tqdm import tqdm\n",
        "from datetime import datetime\n",
        "\n",
        "sys.path.append(\"/content/CosPlace/\")\n",
        "sys.path.append(\"/content/geo_warp/\")\n",
        "import CosPlace\n",
        "from CosPlace import *\n",
        "\n",
        "from geo_warp import *\n",
        "\n",
        "torch.backends.cudnn.benchmark = True  # Provides a speedup\n",
        "print(\"ok\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DTa6drZa52o0"
      },
      "outputs": [],
      "source": [
        "'''\n",
        "train.py and eval.py are the one that we use to run for test and train\n",
        "\n",
        "requirements for geowarp if not executed earlier:\n",
        "faiss_cpu\n",
        "gdown\n",
        "kornia\n",
        "numpy\n",
        "Pillow\n",
        "scikit_learn\n",
        "Shapely\n",
        "torch\n",
        "torchvision\n",
        "tqdm\n",
        "'''"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4sdYefUDELO5",
        "outputId": "a0fa090a-8b39-4780-ad7d-7040c2d4c0c7"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:root:The outputs are being saved in logs/cosface_model/2023-02-01_12-52-46\n"
          ]
        }
      ],
      "source": [
        "from CosPlace import commons\n",
        "\n",
        "dirCosface = \"cosface_model\" #output to save the model \n",
        "start_time = datetime.now()\n",
        "output_folder = f\"logs/{dirCosface}/{start_time.strftime('%Y-%m-%d_%H-%M-%S')}\"\n",
        "commons.make_deterministic(0) #seed \n",
        "'''\n",
        "Make results deterministic. If seed == -1, do not make deterministic.\n",
        "Running your script in a deterministic way might slow it down.\n",
        "Note that for some packages (eg: sklearn's PCA) this function is not enough.\n",
        "'''\n",
        "commons.setup_logging(output_folder, console=None)\n",
        "''' \n",
        "Set up logging files and console output.\n",
        "Creates one file for INFO logs and one for DEBUG logs.\n",
        "Args:\n",
        "- output_folder (str): creates the folder where to save the files.\n",
        "- exist_ok (boolean): if False throw a FileExistsError if output_folder already exists\n",
        "- debug (str):\n",
        "    if == \"debug\" prints on console debug messages and higher\n",
        "    if == \"info\"  prints on console info messages and higher\n",
        "    if == None does not use console (useful when a logger has already been set)\n",
        "- info_filename (str): the name of the info file. if None, don't create info file\n",
        "- debug_filename (str): the name of the debug file. if None, don't create debug file\n",
        "\n",
        "All the args that are not output_folder are not `output_folder` are already assigned by default\n",
        "'''\n",
        "logging.info(f\"The outputs are being saved in {output_folder}\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "mJDPKcD2HCnP"
      },
      "source": [
        "## Train geo_warp"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kJu8uyHDHd38",
        "outputId": "49fafe97-4fa2-4d02-80b3-e72666615f4a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting kornia\n",
            "  Downloading kornia-0.6.9-py2.py3-none-any.whl (569 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m569.1/569.1 KB\u001b[0m \u001b[31m15.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch>=1.9.1 in /usr/local/lib/python3.8/dist-packages (from kornia) (1.13.1+cu116)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from kornia) (21.3)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch>=1.9.1->kornia) (4.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->kornia) (3.0.9)\n",
            "Installing collected packages: kornia\n",
            "Successfully installed kornia-0.6.9\n"
          ]
        }
      ],
      "source": [
        "pip install kornia #required for geo_warp"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_3FYg83TLQ0E"
      },
      "outputs": [],
      "source": [
        "#download pre trained baseline (code fro gmberton/geo_warp)\n",
        "#it is already in the drive so skip this\n",
        "import os\n",
        "import gdown\n",
        "import shutil\n",
        "from google_drive_downloader import GoogleDriveDownloader as gdd\n",
        "\n",
        "gdd.download_file_from_google_drive(file_id=\"1WjfQFS_13uvg_eyefNJWVIKbvy9tSPR-\",\n",
        "                                    dest_path=\"./tmp\", unzip=True)\n",
        "os.remove(\"tmp\")\n",
        "url = \"https://drive.google.com/uc?id=1WjfQFS_13uvg_eyefNJWVIKbvy9tSPR-\"\n",
        "tmp_archive_name = \"tmp.zip\"\n",
        "gdown.download(url, tmp_archive_name, quiet=False)\n",
        "\n",
        "shutil.unpack_archive(tmp_archive_name)\n",
        "os.remove(tmp_archive_name)\n",
        "os.makedirs(\"data\", exist_ok=True)\n",
        "#watch out for  paths\n",
        "shutil.move(\"pretrained_baselines\", \"/content/geo_warp/data/pretrained_baselines\")\n",
        "print(\"ok\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pRSQc3LrJ6Jn",
        "outputId": "63bb651c-8dac-4cac-e0dd-012f41ba221a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  /content/pretrained_dataset/pretrained_baselines.zip\n",
            "   creating: pretrained_baselines/\n",
            "  inflating: pretrained_baselines/alexnet_gem.pth  \n",
            "  inflating: pretrained_baselines/resnet50_gem.pth  \n",
            "  inflating: pretrained_baselines/resnet50_netvlad.pth  \n",
            "  inflating: pretrained_baselines/alexnet_netvlad.pth  \n",
            "  inflating: pretrained_baselines/vgg16_gem.pth  \n",
            "  inflating: pretrained_baselines/vgg16_netvlad.pth  \n",
            "ok\n"
          ]
        }
      ],
      "source": [
        "#unzip the pretrained baseline -> only if the block above is not performed (we have a local copy on our drive)\n",
        "!unzip /content/pretrained_dataset/pretrained_baselines.zip\n",
        "shutil.move(\"/content/pretrained_baselines\", \"/content/geo_warp/data/\")\n",
        "print(\"ok\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dNxdN36fQkTA"
      },
      "outputs": [],
      "source": [
        "#!python /content/geo_warp/train.py -h\n",
        "train_set_folder = os.path.join(\"/content/small/\", \"train\")\n",
        "val_set_folder = os.path.join(\"/content/small/\", \"val\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MndIW71Nd4Ct",
        "outputId": "26faacd8-b848-45f1-9f6a-e55e0a7ba434"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting backpack\n",
            "  Downloading backpack-0.1.tar.gz (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.3/71.3 KB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting simplejson\n",
            "  Downloading simplejson-3.18.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (135 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.5/135.5 KB\u001b[0m \u001b[31m18.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: backpack\n",
            "  Building wheel for backpack (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for backpack: filename=backpack-0.1-py2.py3-none-any.whl size=12449 sha256=450f021c8d211d7352404ccc897cded07627992f449fbcb4e6ef136264e23042\n",
            "  Stored in directory: /root/.cache/pip/wheels/0d/ff/02/bee22c0dd24f523e49f46b7f73a46dc046f0206ced4ce9199e\n",
            "Successfully built backpack\n",
            "Installing collected packages: simplejson, backpack\n",
            "Successfully installed backpack-0.1 simplejson-3.18.1\n"
          ]
        }
      ],
      "source": [
        "!pip install backpack #laplace"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "OHDqs6HqK67B"
      },
      "source": [
        "## Train with sf-xs (small)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ck3Y7zkCLJbx"
      },
      "source": [
        "need to:\n",
        "- add dataset_wrap (local) (OK) ADDED\n",
        "- modify network.py (add geowarp network, homografic, feature extractor)\n",
        "- modify test.py (remove laplace.py)\n",
        "- modify layer.py (feature l2 norm)\n",
        "- add train_geowarp.py (OK) ADDED\n",
        "- modify parser.py (geowarp parameters)\n",
        "- modify commons (get_output_dim)\n",
        "\n",
        "togliere"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "XTYUyWwj_r7t"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "test with no warping: R@1: 50.5, R@5: 66.1, R@10: 70.9, R@20: 75.8\n",
        "\n",
        "test after warping - R@1: 60.2, R@5: 66.1, R@10: 70.9, R@20: 75.8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2rh9taCqHES0",
        "outputId": "61f47f19-10bd-4f57-c073-7e57734b3875"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/content/train_geowarp.py\", line 55, in <module>\n",
            "    args = parser.parse_arguments()\n",
            "  File \"/content/parser.py\", line 96, in parse_arguments\n",
            "    parser.add_argument(\"--multi_scale\", action='store_true', help=\"Use multi scale\")\n",
            "  File \"/usr/lib/python3.8/argparse.py\", line 1398, in add_argument\n",
            "    return self._add_action(action)\n",
            "  File \"/usr/lib/python3.8/argparse.py\", line 1761, in _add_action\n",
            "    self._optionals._add_action(action)\n",
            "  File \"/usr/lib/python3.8/argparse.py\", line 1602, in _add_action\n",
            "    action = super(_ArgumentGroup, self)._add_action(action)\n",
            "  File \"/usr/lib/python3.8/argparse.py\", line 1412, in _add_action\n",
            "    self._check_conflict(action)\n",
            "  File \"/usr/lib/python3.8/argparse.py\", line 1551, in _check_conflict\n",
            "    conflict_handler(action, confl_optionals)\n",
            "  File \"/usr/lib/python3.8/argparse.py\", line 1560, in _handle_conflict_error\n",
            "    raise ArgumentError(action, message % conflict_string)\n",
            "argparse.ArgumentError: argument --multi_scale: conflicting option string: --multi_scale\n"
          ]
        }
      ],
      "source": [
        "!python /content/train_geowarp.py --dataset_folder /content/small --groups_num 1 --epochs_num 3\n",
        "#BE CAREFUL: THE MODEL IS ALREADY TRAINED AND IT IS IN geowarp_model"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "WKr8cxnnRQOD"
      },
      "source": [
        "## Evaluation on sf-xs (small)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "pwLHL4h6RSpR"
      },
      "source": [
        "need to:\n",
        "- add evalGeowarp.py ADDED\n",
        "- modify util (compute_features and compute_recall) -> now there are dataset_util_geowarp and dataset_geoloc ADDED\n",
        "- add dataset_geoloc ADDED\n",
        "- add dataset_util_geowarp ADDED \n",
        "\n",
        "togliere"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "3U9dcAG2_6Ch"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "test with no warping: R@1: 50.5, R@5: 66.1, R@10: 70.9, R@20: 75.8\n",
        "\n",
        "test after warping - R@1: 60.2, R@5: 66.1, R@10: 70.9, R@20: 75.8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IR0jEGFfgRkK",
        "outputId": "dc3409d7-538b-4f28-9739-8dcacf024f7f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 09:02:57   /content/evalGeowarp.py --dataset_folder /content/small/ --resume_model /content/geowarp_model/best_model.pth\n",
            "2023-04-21 09:02:57   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-21 09:02:57   The outputs are being saved in logs/default/2023-04-21_09-02-57\n",
            "2023-04-21 09:03:00   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 09:03:00   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-04-21 09:03:01   Start testing\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [03:42<00:00,  7.66it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:16<00:00, 59.08it/s]\n",
            "2023-04-21 09:07:01   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:44<00:00,  6.09it/s]\n",
            "2023-04-21 09:09:45   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 50.5, R@5: 66.1, R@10: 70.9, R@20: 75.8\n",
            "2023-04-21 09:09:45     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 60.2, R@5: 66.1, R@10: 70.9, R@20: 75.8\n",
            "2023-04-21 09:09:45   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small/ --resume_model /content/geowarp_model/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "N8-gQRxsDJsc"
      },
      "source": [
        "## Evaluation on tokyo-xs "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "5I5_VbHPAGRr"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "Test without warping: < test - #q: 315; #db: 12771 >: R@1: 70.5, R@5: 83.2, R@10: 88.9, R@20: 91.7\n",
        "\n",
        "Test after warping: < test - #q: 315; #db: 12771 >: R@1: 76.2, R@5: 83.2, R@10: 88.9, R@20: 91.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_HLaXgJNDNAN",
        "outputId": "3702e5ae-56c1-4f67-ef9b-c972661b40d6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 09:41:52   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --resume_model /content/geowarp_model/best_model.pth\n",
            "2023-04-21 09:41:52   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 09:41:52   The outputs are being saved in logs/default/2023-04-21_09-41-52\n",
            "2023-04-21 09:41:55   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 09:41:55   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-04-21 09:41:55   Start testing\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:10<00:00,  6.14it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:04<00:00, 65.45it/s]\n",
            "2023-04-21 09:44:10   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:57<00:00,  5.45it/s]\n",
            "2023-04-21 09:45:08   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 70.5, R@5: 83.2, R@10: 88.9, R@20: 91.7\n",
            "2023-04-21 09:45:08     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 76.2, R@5: 83.2, R@10: 88.9, R@20: 91.7\n",
            "2023-04-21 09:45:08   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs  --resume_model /content/geowarp_model/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Zyf-DE8DDPzi"
      },
      "source": [
        "## Evaluation on tokyo night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Pwy9u0SHAPYX"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "Test without warping: < test - #q: 105; #db: 12771 >: R@1: 52.4, R@5: 70.5, R@10: 81.0, R@20: 84.8\n",
        "\n",
        "\n",
        "Test after warping: < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 70.5, R@10: 81.0, R@20: 84.8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Rp5x7FmDSld",
        "outputId": "305e4a75-e3f9-4d98-c08b-9577b572b105"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 09:45:12   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --resume_model /content/geowarp_model/best_model.pth\n",
            "2023-04-21 09:45:12   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 09:45:12   The outputs are being saved in logs/default/2023-04-21_09-45-12\n",
            "2023-04-21 09:45:14   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 09:45:14   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-04-21 09:45:15   Start testing\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:06<00:00,  6.32it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 50.62it/s]\n",
            "2023-04-21 09:47:23   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:19<00:00,  5.51it/s]\n",
            "2023-04-21 09:47:42   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 52.4, R@5: 70.5, R@10: 81.0, R@20: 84.8\n",
            "2023-04-21 09:47:42     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 70.5, R@10: 81.0, R@20: 84.8\n",
            "2023-04-21 09:47:42   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night  --resume_model /content/geowarp_model/best_model.pth #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "VxBTJ91p31Mp"
      },
      "source": [
        "# Step 6: Data Augmentation - Only on tokyo night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "61HwDEJbJLEK"
      },
      "source": [
        "## CosPlace"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "a2Vp6fwuAWfE"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "R@1: 52.4, R@5: 70.5, R@10: 78.1, R@20: 87.6"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TsM80RWZ1dZI",
        "outputId": "91962b28-5860-4757-d75f-b539cf03876d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 09:48:06   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth --night_test True\n",
            "2023-04-21 09:48:06   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_cosface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=True, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 09:48:06   The outputs are being saved in logs/default/2023-04-21_09-48-06\n",
            "2023-04-21 09:48:07   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 09:48:07   Loading model from /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:36<00:00,  5.12it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 50.74it/s]\n",
            "2023-04-21 09:50:47   < test - #q: 105; #db: 12771 >: R@1: 52.4, R@5: 70.5, R@10: 78.1, R@20: 87.6\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth --night_test True #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "PpNeBRVaJQDN"
      },
      "source": [
        "## SphereFace"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "VnXzrNUYJUev"
      },
      "source": [
        "Results 03/02/2023:\n",
        "\n",
        "R@1: 46.7, R@5: 72.4, R@10: 79.0, R@20: 83.8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KN0I8JMYJUzR",
        "outputId": "9b800347-6f9b-4c7c-ae17-67e98e9a5c58"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 09:50:51   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/trained_with_sphereface/best_model.pth --night_test True\n",
            "2023-04-21 09:50:51   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_sphereface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=True, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 09:50:51   The outputs are being saved in logs/default/2023-04-21_09-50-51\n",
            "2023-04-21 09:50:51   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 09:50:51   Loading model from /content/logs/content/logs/default/trained_with_sphereface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:32<00:00,  5.23it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 37.52it/s]\n",
            "2023-04-21 09:53:28   < test - #q: 105; #db: 12771 >: R@1: 46.7, R@5: 70.5, R@10: 76.2, R@20: 87.6\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/trained_with_sphereface/best_model.pth --night_test True #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "4JcCuU3TJvQ8"
      },
      "source": [
        "## ArcFace"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "hSIQnmKkJweU"
      },
      "source": [
        "Results 03/02/2023: \n",
        "\n",
        "R@1: 51.4, R@5: 67.6, R@10: 77.1, R@20: 81.0\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4hZ8N640JxIM",
        "outputId": "86e5bbf5-6e53-4ecb-fa6d-97ce188deee3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 09:53:32   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/trained_with_arcface/best_model.pth --night_test True\n",
            "2023-04-21 09:53:32   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_arcface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=True, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 09:53:32   The outputs are being saved in logs/default/2023-04-21_09-53-32\n",
            "2023-04-21 09:53:32   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 09:53:32   Loading model from /content/logs/content/logs/default/trained_with_arcface/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:32<00:00,  5.23it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 33.08it/s]\n",
            "2023-04-21 09:56:10   < test - #q: 105; #db: 12771 >: R@1: 51.4, R@5: 67.6, R@10: 77.1, R@20: 81.0\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/trained_with_arcface/best_model.pth --night_test True #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "4LAE5-5W-aPU"
      },
      "source": [
        "# Step 7: Alternative backbone"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "loRtcKXh-vzI"
      },
      "source": [
        "Be sure that requirements.txt have been already installed otherwise run the following command:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-uoHK2Wo-fCC",
        "outputId": "afa9d835-0a93-4ea6-93c7-f4ac3f8f2d3c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "\u001b[31mERROR: Could not find a version that satisfies the requirement requirements.txt (from versions: none)\u001b[0m\u001b[31m\n",
            "\u001b[0mHINT: You are attempting to install a package literally named \"requirements.txt\" (which cannot exist). Consider using the '-r' flag to install the packages listed in requirements.txt\n",
            "\u001b[31mERROR: No matching distribution found for requirements.txt\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip install requirements.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IuGS3lF9_QAK",
        "outputId": "b192aff4-e74b-4b96-f2ca-8b4114586321"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'CosPlace'...\n",
            "remote: Enumerating objects: 224, done.\u001b[K\n",
            "remote: Counting objects: 100% (111/111), done.\u001b[K\n",
            "remote: Compressing objects: 100% (36/36), done.\u001b[K\n",
            "remote: Total 224 (delta 84), reused 85 (delta 75), pack-reused 113\u001b[K\n",
            "Receiving objects: 100% (224/224), 62.60 KiB | 660.00 KiB/s, done.\n",
            "Resolving deltas: 100% (125/125), done.\n"
          ]
        }
      ],
      "source": [
        "#If not downloaded, download cosplace code\n",
        "# download code of CosPlace\n",
        "!git clone \"https://github.com/gmberton/CosPlace\" \n",
        "#!rm -r \"/content/CosPlace\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XelerOT2_XD9"
      },
      "outputs": [],
      "source": [
        "#import code\n",
        "import os\n",
        "import sys\n",
        "import torch\n",
        "import logging\n",
        "import multiprocessing\n",
        "import numpy as np\n",
        "#import torchvision.transforms as T\n",
        "from tqdm import tqdm\n",
        "from datetime import datetime\n",
        "\n",
        "sys.path.append(\"/content/CosPlace/\")\n",
        "import CosPlace\n",
        "from CosPlace import *\n",
        "\n",
        "torch.backends.cudnn.benchmark = True  # Provides a speedup"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "1q71PHGJskQH"
      },
      "source": [
        "Modified file(s):\n",
        "- network.py (added convnext_tiny) - in doubt i modified netwrok 2 too\n",
        "- parser (added convnext in backbone, adjusted optim)\n",
        "- train_alternativebackbone (removed grl_param)\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "GjOS8lYEuUpl"
      },
      "source": [
        "## Train sf-xs (small) with Convnext_tiny"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "gKSgU5jzAeoy"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "R@1: 21.6, R@5: 36.7, R@10: 44.1, R@20: 50.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YswqEumDuT57",
        "outputId": "04119a35-9e29-4f05-dffe-8d0bffa21b2b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-02-03 21:03:38   /content/train_alternativebackbone.py --dataset_folder /content/small --groups_num 1 --iterations_per_epoch 1000 --epochs_num 3 --backbone convnext_tiny\n",
            "2023-02-03 21:03:38   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='convnext_tiny', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=3, fc_output_dim=512, features_wise_w=10, grl_param=None, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=1000, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method=None, night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=8, optim='adam', positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[1, 2, 5, 10], source_dir=None, ss_w=1, target_dir=None, test_method='hard_resize', test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val')\n",
            "2023-02-03 21:03:38   The outputs are being saved in logs/default/2023-02-03_21-03-38\n",
            "2023-02-03 21:03:38   Gradient Reversal Layer is disabled\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ConvNeXt_Tiny_Weights.IMAGENET1K_V1`. You can also use `weights=ConvNeXt_Tiny_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-02-03 21:03:38   This network is pretrained with Imagenet\n",
            "2023-02-03 21:03:39   Train last layer of ConvNext, freeze the previous ones\n",
            "2023-02-03 21:03:39   There are 1 GPUs and 2 CPUs.\n",
            "2023-02-03 21:03:41   ADAM OPTIMIZER\n",
            "2023-02-03 21:03:41   Using cached dataset cache/small_M10_N5_mipc10.torch\n",
            "2023-02-03 21:03:41   Using cosface loss function.\n",
            "2023-02-03 21:03:41   Using 1 groups\n",
            "2023-02-03 21:03:41   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-02-03 21:03:41   The 1 groups have respectively the following number of images [59650]\n",
            "2023-02-03 21:03:42   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-02-03 21:03:42   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-03 21:03:42   Start training ...\n",
            "2023-02-03 21:03:42   There are 5965 classes for the first group, each epoch has 1000 iterations with batch_size 32, therefore the model sees each class (on average) 5.4 times per epoch\n",
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  1%|▍                                                             | 8/1000 [00:15<31:48,  1.92s/it]\n",
            "2023-02-03 21:03:58   \n",
            "Traceback (most recent call last):\n",
            "  File \"/content/train_alternativebackbone.py\", line 163, in <module>\n",
            "    epoch_losses = np.append(epoch_losses, loss.item())\n",
            "KeyboardInterrupt\n",
            "\n",
            "2023-02-03 21:03:58   Experiment finished (with some errors)\n",
            "^C\n"
          ]
        }
      ],
      "source": [
        "!python /content/train_alternativebackbone.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --backbone convnext_tiny --grl_param 0.3\n",
        "#BE CAREFUL: THE MODEL IS ALREADY TRAINED AND IT IS IN geowarp_model BUT IT DOES NOT WORKS IF YOU DON'T TRAIN"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "atXGMzAzMsmg"
      },
      "source": [
        "## Train on sf-xs with Efficient net v2s"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "WEVkMBvzXCwg"
      },
      "source": [
        "Results 04/02/2023\n",
        "\n",
        "SCARICARE MODELLO"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JAfeIza3M1dm",
        "outputId": "bfefbe52-10fb-4f3a-d610-96f4e08ddd41"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-02-04 16:56:55   /content/train.py --dataset_folder /content/small --groups_num 1 --backbone efficientnet_v2_s --grl_param 0.3 --source_dir /content/small --target_dir /content/night_target\n",
            "2023-02-04 16:56:55   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='efficientnet_v2_s', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=50, fc_output_dim=512, features_wise_w=10, grl_param=0.3, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10000, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method='avg', night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=8, optim='adam', positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resize=[480, 640], resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], source_dir='/content/small', ss_w=1, target_dir='/content/night_target', test_method='hard_resize', test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val', wd=None)\n",
            "2023-02-04 16:56:55   The outputs are being saved in logs/default/2023-02-04_16-56-55\n",
            "2023-02-04 16:56:55   Gradient Reversal Layer is enabled with parameter = 0.3\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:00<00:00, 95.5MB/s]\n",
            "2023-02-04 16:56:56   Using efficient net v2s\n",
            "2023-02-04 16:56:56   Model uses weights IMAGENET1K_V1\n",
            "2023-02-04 16:56:57   Train last two layers of EfficientNet, freeze the previous ones\n",
            "2023-02-04 16:56:57   There are 1 GPUs and 12 CPUs.\n",
            "2023-02-04 16:57:02   Cached dataset cache/small_M10_N5_mipc10.torch does not exist, I'll create it now.\n",
            "2023-02-04 16:57:02   Searching training images in /content/small/train\n",
            "2023-02-04 16:57:02   Found 59650 images\n",
            "2023-02-04 16:57:02   For each image, get its UTM east, UTM north and heading from its path\n",
            "2023-02-04 16:57:02   For each image, get class and group to which it belongs\n",
            "2023-02-04 16:57:02   Group together images belonging to the same class\n",
            "2023-02-04 16:57:02   Group together classes belonging to the same group\n",
            "2023-02-04 16:57:03   GrlDataset has 3 domain classes\n",
            "2023-02-04 16:57:03   Using cosface loss function.\n",
            "2023-02-04 16:57:03   Using 1 groups\n",
            "2023-02-04 16:57:03   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-02-04 16:57:03   The 1 groups have respectively the following number of images [59650]\n",
            "2023-02-04 16:57:03   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-02-04 16:57:03   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-04 16:57:03   Start training ...\n",
            "2023-02-04 16:57:03   There are 5965 classes for the first group, each epoch has 10000 iterations with batch_size 32, therefore the model sees each class (on average) 53.6 times per epoch\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [53:25<00:00,  3.12it/s]\n",
            "2023-02-04 17:50:30   Epoch 00 in 0:53:26, loss = 8.8910\n",
            "2023-02-04 17:50:30   Average GRL epoch loss (* alpha = 0.3): 0.1810\n",
            "2023-02-04 17:50:30   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:22<00:00, 21.88it/s]\n",
            "2023-02-04 17:50:53   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [05:34<00:00, 23.89it/s]\n",
            "2023-02-04 17:56:27   Calculating recalls\n",
            "2023-02-04 17:56:29   Epoch 00 in 0:59:25, < val - #q: 7993; #db: 8015 >: R@1: 83.2, R@5: 91.1\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [53:08<00:00,  3.14it/s]\n",
            "2023-02-04 18:49:39   Epoch 01 in 0:53:09, loss = 3.6255\n",
            "2023-02-04 18:49:39   Average GRL epoch loss (* alpha = 0.3): 0.3582\n",
            "2023-02-04 18:49:39   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:23<00:00, 21.71it/s]\n",
            "2023-02-04 18:50:02   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [05:36<00:00, 23.78it/s]\n",
            "2023-02-04 18:55:38   Calculating recalls\n",
            "2023-02-04 18:55:39   Epoch 01 in 0:59:09, < val - #q: 7993; #db: 8015 >: R@1: 86.7, R@5: 93.3\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [53:08<00:00,  3.14it/s]\n",
            "2023-02-04 19:48:50   Epoch 02 in 0:53:09, loss = 2.4832\n",
            "2023-02-04 19:48:50   Average GRL epoch loss (* alpha = 0.3): 0.5414\n",
            "2023-02-04 19:48:50   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:22<00:00, 22.34it/s]\n",
            "2023-02-04 19:49:12   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [05:35<00:00, 23.79it/s]\n",
            "2023-02-04 19:54:48   Calculating recalls\n",
            "2023-02-04 19:54:50   Epoch 02 in 0:59:09, < val - #q: 7993; #db: 8015 >: R@1: 87.7, R@5: 93.7\n",
            " 67%|███████████████████████████████████████                   | 6743/10000 [35:54<16:52,  3.22it/s]Exception ignored in: <generator object tqdm.__iter__ at 0x7f3029a72190>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/tqdm/std.py\", line 1210, in __iter__\n",
            "    self.close()\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/tqdm/std.py\", line 1316, in close\n",
            "    self.display(pos=0)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/tqdm/std.py\", line 1509, in display\n",
            "    self.sp(self.__str__() if msg is None else msg)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/tqdm/std.py\", line 1165, in __str__\n",
            "    return self.format_meter(**self.format_dict)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/tqdm/std.py\", line 507, in format_meter\n",
            "    l_bar += '{0:3.0f}%|'.format(percentage)\n",
            "KeyboardInterrupt: \n",
            "2023-02-04 20:30:46   \n",
            "Traceback (most recent call last):\n",
            "  File \"/content/train.py\", line 196, in <module>\n",
            "    domain_adapt_output = model(domain_adapt_images,force_grl=True)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\", line 1194, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/content/model/network.py\", line 49, in forward\n",
            "    x = self.backbone(x)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\", line 1194, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/torch/nn/modules/container.py\", line 204, in forward\n",
            "    input = module(input)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\", line 1194, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/torch/nn/modules/container.py\", line 204, in forward\n",
            "    input = module(input)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\", line 1194, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/torchvision/models/efficientnet.py\", line 229, in forward\n",
            "    result += input\n",
            "KeyboardInterrupt\n",
            "\n",
            "2023-02-04 20:30:46   Experiment finished (with some errors)\n",
            "^C\n"
          ]
        }
      ],
      "source": [
        "!python /content/train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --backbone efficientnet_v2_s --grl_param 0.3 --source_dir /content/small --target_dir /content/night_target "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y1H9VvIdVgzi",
        "outputId": "bdf0caf4-20b2-4b9f-a6ed-374aedf6d494"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-02-05 06:18:50   /content/train.py --dataset_folder /content/small --groups_num 1 --epochs_num 1 --iterations_per_epoch 10 --backbone efficientnet_v2_s --grl_param 0.3 --source_dir /content/small --target_dir /content/night_target\n",
            "2023-02-05 06:18:50   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='efficientnet_v2_s', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=1, fc_output_dim=512, features_wise_w=10, grl_param=0.3, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method='avg', night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=8, optim='adam', positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resize=[480, 640], resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], source_dir='/content/small', ss_w=1, target_dir='/content/night_target', test_method='hard_resize', test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val', wd=None)\n",
            "2023-02-05 06:18:50   The outputs are being saved in logs/default/2023-02-05_06-18-50\n",
            "2023-02-05 06:18:50   Gradient Reversal Layer is enabled with parameter = 0.3\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:00<00:00, 102MB/s]\n",
            "2023-02-05 06:18:53   Using efficient net v2s\n",
            "2023-02-05 06:18:53   Model uses weights IMAGENET1K_V1\n",
            "2023-02-05 06:18:54   Train last two layers of EfficientNet, freeze the previous ones\n",
            "2023-02-05 06:18:55   There are 1 GPUs and 2 CPUs.\n",
            "2023-02-05 06:18:56   Cached dataset cache/small_M10_N5_mipc10.torch does not exist, I'll create it now.\n",
            "2023-02-05 06:18:56   Searching training images in /content/small/train\n",
            "2023-02-05 06:18:56   Found 59650 images\n",
            "2023-02-05 06:18:56   For each image, get its UTM east, UTM north and heading from its path\n",
            "2023-02-05 06:18:57   For each image, get class and group to which it belongs\n",
            "2023-02-05 06:18:57   Group together images belonging to the same class\n",
            "2023-02-05 06:18:57   Group together classes belonging to the same group\n",
            "2023-02-05 06:18:57   GrlDataset has 3 domain classes\n",
            "2023-02-05 06:18:57   Using cosface loss function.\n",
            "2023-02-05 06:18:57   Using 1 groups\n",
            "2023-02-05 06:18:57   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-02-05 06:18:57   The 1 groups have respectively the following number of images [59650]\n",
            "2023-02-05 06:18:58   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-02-05 06:18:58   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-05 06:18:58   Start training ...\n",
            "2023-02-05 06:18:58   There are 5965 classes for the first group, each epoch has 10 iterations with batch_size 32, therefore the model sees each class (on average) 0.1 times per epoch\n",
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████████| 10/10 [00:34<00:00,  3.49s/it]\n",
            "2023-02-05 06:19:34   Epoch 00 in 0:00:36, loss = 37.7771\n",
            "2023-02-05 06:19:34   Average GRL epoch loss (* alpha = 0.3): 0.2119\n",
            "2023-02-05 06:19:34   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [02:09<00:00,  3.86it/s]\n",
            "2023-02-05 06:21:44   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [06:00<00:00, 22.18it/s]\n",
            "2023-02-05 06:27:45   Calculating recalls\n",
            "2023-02-05 06:27:46   Epoch 00 in 0:08:48, < val - #q: 7993; #db: 8015 >: R@1: 30.4, R@5: 45.9\n",
            "2023-02-05 06:27:48   Trained for 01 epochs, in total in 0:08:57\n",
            "2023-02-05 06:27:48   Now testing on the test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-05 06:27:48   Extracting database descriptors for evaluation/testing\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [07:09<00:00,  3.95it/s]\n",
            "2023-02-05 06:34:58   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:47<00:00, 21.24it/s]\n",
            "2023-02-05 06:35:45   Calculating recalls\n",
            "2023-02-05 06:35:46   < test - #q: 1000; #db: 27191 >: R@1: 11.2, R@5: 25.9, R@10: 32.1, R@20: 38.8\n",
            "2023-02-05 06:35:46   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/train.py --dataset_folder /content/small --groups_num 1 --epochs_num 1 --iterations_per_epoch 10 --backbone efficientnet_v2_s --grl_param 0.3 --source_dir /content/small --target_dir /content/night_target "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gyZ_YdBSV2WG",
        "outputId": "399caca9-a010-4c75-a3b8-af0b47672935"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-02-05 06:36:25   /content/evalGRL.py --dataset_folder /content/small/ --resume_model /content/logs/default/2023-02-05_06-18-50/best_model.pth --grl_param 0.3\n",
            "2023-02-05 06:36:25   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='ResNet18', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small/', device='cuda', epochs_num=50, fc_output_dim=512, features_wise_w=10, grl_param=0.3, groups_num=8, hue=0.5, infer_batch_size=16, iterations_per_epoch=10000, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method='avg', night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=8, optim='adam', positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resize=[480, 640], resume_model='/content/logs/default/2023-02-05_06-18-50/best_model.pth', resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], source_dir=None, ss_w=1, target_dir=None, test_method='hard_resize', test_set_folder='/content/small/test', use_amp16=False, wd=None)\n",
            "2023-02-05 06:36:25   The outputs are being saved in logs/default/2023-02-05_06-36-25\n",
            "2023-02-05 06:36:25   Train only layer3 and layer4 of the ResNet18, freeze the previous ones\n",
            "2023-02-05 06:36:31   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-05 06:36:32   \n",
            "Traceback (most recent call last):\n",
            "  File \"/content/evalGRL.py\", line 53, in <module>\n",
            "    model.load_state_dict(best_model_state_dict)\n",
            "  File \"/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\", line 1671, in load_state_dict\n",
            "    raise RuntimeError('Error(s) in loading state_dict for {}:\\n\\t{}'.format(\n",
            "RuntimeError: Error(s) in loading state_dict for GeoLocalizationNet:\n",
            "\tMissing key(s) in state_dict: \"backbone.0.weight\", \"backbone.1.weight\", \"backbone.1.bias\", \"backbone.1.running_mean\", \"backbone.1.running_var\", \"backbone.4.0.conv1.weight\", \"backbone.4.0.bn1.weight\", \"backbone.4.0.bn1.bias\", \"backbone.4.0.bn1.running_mean\", \"backbone.4.0.bn1.running_var\", \"backbone.4.0.conv2.weight\", \"backbone.4.0.bn2.weight\", \"backbone.4.0.bn2.bias\", \"backbone.4.0.bn2.running_mean\", \"backbone.4.0.bn2.running_var\", \"backbone.4.1.conv1.weight\", \"backbone.4.1.bn1.weight\", \"backbone.4.1.bn1.bias\", \"backbone.4.1.bn1.running_mean\", \"backbone.4.1.bn1.running_var\", \"backbone.4.1.conv2.weight\", \"backbone.4.1.bn2.weight\", \"backbone.4.1.bn2.bias\", \"backbone.4.1.bn2.running_mean\", \"backbone.4.1.bn2.running_var\", \"backbone.5.0.conv1.weight\", \"backbone.5.0.bn1.weight\", \"backbone.5.0.bn1.bias\", \"backbone.5.0.bn1.running_mean\", \"backbone.5.0.bn1.running_var\", \"backbone.5.0.conv2.weight\", \"backbone.5.0.bn2.weight\", \"backbone.5.0.bn2.bias\", \"backbone.5.0.bn2.running_mean\", \"backbone.5.0.bn2.running_var\", \"backbone.5.0.downsample.0.weight\", \"backbone.5.0.downsample.1.weight\", \"backbone.5.0.downsample.1.bias\", \"backbone.5.0.downsample.1.running_mean\", \"backbone.5.0.downsample.1.running_var\", \"backbone.5.1.conv1.weight\", \"backbone.5.1.bn1.weight\", \"backbone.5.1.bn1.bias\", \"backbone.5.1.bn1.running_mean\", \"backbone.5.1.bn1.running_var\", \"backbone.5.1.conv2.weight\", \"backbone.5.1.bn2.weight\", \"backbone.5.1.bn2.bias\", \"backbone.5.1.bn2.running_mean\", \"backbone.5.1.bn2.running_var\", \"backbone.6.0.conv1.weight\", \"backbone.6.0.bn1.weight\", \"backbone.6.0.bn1.bias\", \"backbone.6.0.bn1.running_mean\", \"backbone.6.0.bn1.running_var\", \"backbone.6.0.conv2.weight\", \"backbone.6.0.bn2.weight\", \"backbone.6.0.bn2.bias\", \"backbone.6.0.bn2.running_mean\", \"backbone.6.0.bn2.running_var\", \"backbone.6.0.downsample.0.weight\", \"backbone.6.0.downsample.1.weight\", \"backbone.6.0.downsample.1.bias\", \"backbone.6.0.downsample.1.running_mean\", \"backbone.6.0.downsample.1.running_var\", \"backbone.6.1.conv1.weight\", \"backbone.6.1.bn1.weight\", \"backbone.6.1.bn1.bias\", \"backbone.6.1.bn1.running_mean\", \"backbone.6.1.bn1.running_var\", \"backbone.6.1.conv2.weight\", \"backbone.6.1.bn2.weight\", \"backbone.6.1.bn2.bias\", \"backbone.6.1.bn2.running_mean\", \"backbone.6.1.bn2.running_var\", \"backbone.7.0.conv1.weight\", \"backbone.7.0.bn1.weight\", \"backbone.7.0.bn1.bias\", \"backbone.7.0.bn1.running_mean\", \"backbone.7.0.bn1.running_var\", \"backbone.7.0.conv2.weight\", \"backbone.7.0.bn2.weight\", \"backbone.7.0.bn2.bias\", \"backbone.7.0.bn2.running_mean\", \"backbone.7.0.bn2.running_var\", \"backbone.7.0.downsample.0.weight\", \"backbone.7.0.downsample.1.weight\", \"backbone.7.0.downsample.1.bias\", \"backbone.7.0.downsample.1.running_mean\", \"backbone.7.0.downsample.1.running_var\", \"backbone.7.1.conv1.weight\", \"backbone.7.1.bn1.weight\", \"backbone.7.1.bn1.bias\", \"backbone.7.1.bn1.running_mean\", \"backbone.7.1.bn1.running_var\", \"backbone.7.1.conv2.weight\", \"backbone.7.1.bn2.weight\", \"backbone.7.1.bn2.bias\", \"backbone.7.1.bn2.running_mean\", \"backbone.7.1.bn2.running_var\". \n",
            "\tUnexpected key(s) in state_dict: \"backbone.0.0.weight\", \"backbone.0.1.weight\", \"backbone.0.1.bias\", \"backbone.0.1.running_mean\", \"backbone.0.1.running_var\", \"backbone.0.1.num_batches_tracked\", \"backbone.1.0.block.0.0.weight\", \"backbone.1.0.block.0.1.weight\", \"backbone.1.0.block.0.1.bias\", \"backbone.1.0.block.0.1.running_mean\", \"backbone.1.0.block.0.1.running_var\", \"backbone.1.0.block.0.1.num_batches_tracked\", \"backbone.1.1.block.0.0.weight\", \"backbone.1.1.block.0.1.weight\", \"backbone.1.1.block.0.1.bias\", \"backbone.1.1.block.0.1.running_mean\", \"backbone.1.1.block.0.1.running_var\", \"backbone.1.1.block.0.1.num_batches_tracked\", \"backbone.2.0.block.0.0.weight\", \"backbone.2.0.block.0.1.weight\", \"backbone.2.0.block.0.1.bias\", \"backbone.2.0.block.0.1.running_mean\", \"backbone.2.0.block.0.1.running_var\", \"backbone.2.0.block.0.1.num_batches_tracked\", \"backbone.2.0.block.1.0.weight\", \"backbone.2.0.block.1.1.weight\", \"backbone.2.0.block.1.1.bias\", \"backbone.2.0.block.1.1.running_mean\", \"backbone.2.0.block.1.1.running_var\", \"backbone.2.0.block.1.1.num_batches_tracked\", \"backbone.2.1.block.0.0.weight\", \"backbone.2.1.block.0.1.weight\", \"backbone.2.1.block.0.1.bias\", \"backbone.2.1.block.0.1.running_mean\", \"backbone.2.1.block.0.1.running_var\", \"backbone.2.1.block.0.1.num_batches_tracked\", \"backbone.2.1.block.1.0.weight\", \"backbone.2.1.block.1.1.weight\", \"backbone.2.1.block.1.1.bias\", \"backbone.2.1.block.1.1.running_mean\", \"backbone.2.1.block.1.1.running_var\", \"backbone.2.1.block.1.1.num_batches_tracked\", \"backbone.2.2.block.0.0.weight\", \"backbone.2.2.block.0.1.weight\", \"backbone.2.2.block.0.1.bias\", \"backbone.2.2.block.0.1.running_mean\", \"backbone.2.2.block.0.1.running_var\", \"backbone.2.2.block.0.1.num_batches_tracked\", \"backbone.2.2.block.1.0.weight\", \"backbone.2.2.block.1.1.weight\", \"backbone.2.2.block.1.1.bias\", \"backbone.2.2.block.1.1.running_mean\", \"backbone.2.2.block.1.1.running_var\", \"backbone.2.2.block.1.1.num_batches_tracked\", \"backbone.2.3.block.0.0.weight\", \"backbone.2.3.block.0.1.weight\", \"backbone.2.3.block.0.1.bias\", \"backbone.2.3.block.0.1.running_mean\", \"backbone.2.3.block.0.1.running_var\", \"backbone.2.3.block.0.1.num_batches_tracked\", \"backbone.2.3.block.1.0.weight\", \"backbone.2.3.block.1.1.weight\", \"backbone.2.3.block.1.1.bias\", \"backbone.2.3.block.1.1.running_mean\", \"backbone.2.3.block.1.1.running_var\", \"backbone.2.3.block.1.1.num_batches_tracked\", \"backbone.3.0.block.0.0.weight\", \"backbone.3.0.block.0.1.weight\", \"backbone.3.0.block.0.1.bias\", \"backbone.3.0.block.0.1.running_mean\", \"backbone.3.0.block.0.1.running_var\", \"backbone.3.0.block.0.1.num_batches_tracked\", \"backbone.3.0.block.1.0.weight\", \"backbone.3.0.block.1.1.weight\", \"backbone.3.0.block.1.1.bias\", \"backbone.3.0.block.1.1.running_mean\", \"backbone.3.0.block.1.1.running_var\", \"backbone.3.0.block.1.1.num_batches_tracked\", \"backbone.3.1.block.0.0.weight\", \"backbone.3.1.block.0.1.weight\", \"backbone.3.1.block.0.1.bias\", \"backbone.3.1.block.0.1.running_mean\", \"backbone.3.1.block.0.1.running_var\", \"backbone.3.1.block.0.1.num_batches_tracked\", \"backbone.3.1.block.1.0.weight\", \"backbone.3.1.block.1.1.weight\", \"backbone.3.1.block.1.1.bias\", \"backbone.3.1.block.1.1.running_mean\", \"backbone.3.1.block.1.1.running_var\", \"backbone.3.1.block.1.1.num_batches_tracked\", \"backbone.3.2.block.0.0.weight\", \"backbone.3.2.block.0.1.weight\", \"backbone.3.2.block.0.1.bias\", \"backbone.3.2.block.0.1.running_mean\", \"backbone.3.2.block.0.1.running_var\", \"backbone.3.2.block.0.1.num_batches_tracked\", \"backbone.3.2.block.1.0.weight\", \"backbone.3.2.block.1.1.weight\", \"backbone.3.2.block.1.1.bias\", \"backbone.3.2.block.1.1.running_mean\", \"backbone.3.2.block.1.1.running_var\", \"backbone.3.2.block.1.1.num_batches_tracked\", \"backbone.3.3.block.0.0.weight\", \"backbone.3.3.block.0.1.weight\", \"backbone.3.3.block.0.1.bias\", \"backbone.3.3.block.0.1.running_mean\", \"backbone.3.3.block.0.1.running_var\", \"backbone.3.3.block.0.1.num_batches_tracked\", \"backbone.3.3.block.1.0.weight\", \"backbone.3.3.block.1.1.weight\", \"backbone.3.3.block.1.1.bias\", \"backbone.3.3.block.1.1.running_mean\", \"backbone.3.3.block.1.1.running_var\", \"backbone.3.3.block.1.1.num_batches_tracked\", \"backbone.4.2.block.0.0.weight\", \"backbone.4.2.block.0.1.weight\", \"backbone.4.2.block.0.1.bias\", \"backbone.4.2.block.0.1.running_mean\", \"backbone.4.2.block.0.1.running_var\", \"backbone.4.2.block.0.1.num_batches_tracked\", \"backbone.4.2.block.1.0.weight\", \"backbone.4.2.block.1.1.weight\", \"backbone.4.2.block.1.1.bias\", \"backbone.4.2.block.1.1.running_mean\", \"backbone.4.2.block.1.1.running_var\", \"backbone.4.2.block.1.1.num_batches_tracked\", \"backbone.4.2.block.2.fc1.weight\", \"backbone.4.2.block.2.fc1.bias\", \"backbone.4.2.block.2.fc2.weight\", \"backbone.4.2.block.2.fc2.bias\", \"backbone.4.2.block.3.0.weight\", \"backbone.4.2.block.3.1.weight\", \"backbone.4.2.block.3.1.bias\", \"backbone.4.2.block.3.1.running_mean\", \"backbone.4.2.block.3.1.running_var\", \"backbone.4.2.block.3.1.num_batches_tracked\", \"backbone.4.3.block.0.0.weight\", \"backbone.4.3.block.0.1.weight\", \"backbone.4.3.block.0.1.bias\", \"backbone.4.3.block.0.1.running_mean\", \"backbone.4.3.block.0.1.running_var\", \"backbone.4.3.block.0.1.num_batches_tracked\", \"backbone.4.3.block.1.0.weight\", \"backbone.4.3.block.1.1.weight\", \"backbone.4.3.block.1.1.bias\", \"backbone.4.3.block.1.1.running_mean\", \"backbone.4.3.block.1.1.running_var\", \"backbone.4.3.block.1.1.num_batches_tracked\", \"backbone.4.3.block.2.fc1.weight\", \"backbone.4.3.block.2.fc1.bias\", \"backbone.4.3.block.2.fc2.weight\", \"backbone.4.3.block.2.fc2.bias\", \"backbone.4.3.block.3.0.weight\", \"backbone.4.3.block.3.1.weight\", \"backbone.4.3.block.3.1.bias\", \"backbone.4.3.block.3.1.running_mean\", \"backbone.4.3.block.3.1.running_var\", \"backbone.4.3.block.3.1.num_batches_tracked\", \"backbone.4.4.block.0.0.weight\", \"backbone.4.4.block.0.1.weight\", \"backbone.4.4.block.0.1.bias\", \"backbone.4.4.block.0.1.running_mean\", \"backbone.4.4.block.0.1.running_var\", \"backbone.4.4.block.0.1.num_batches_tracked\", \"backbone.4.4.block.1.0.weight\", \"backbone.4.4.block.1.1.weight\", \"backbone.4.4.block.1.1.bias\", \"backbone.4.4.block.1.1.running_mean\", \"backbone.4.4.block.1.1.running_var\", \"backbone.4.4.block.1.1.num_batches_tracked\", \"backbone.4.4.block.2.fc1.weight\", \"backbone.4.4.block.2.fc1.bias\", \"backbone.4.4.block.2.fc2.weight\", \"backbone.4.4.block.2.fc2.bias\", \"backbone.4.4.block.3.0.weight\", \"backbone.4.4.block.3.1.weight\", \"backbone.4.4.block.3.1.bias\", \"backbone.4.4.block.3.1.running_mean\", \"backbone.4.4.block.3.1.running_var\", \"backbone.4.4.block.3.1.num_batches_tracked\", \"backbone.4.5.block.0.0.weight\", \"backbone.4.5.block.0.1.weight\", \"backbone.4.5.block.0.1.bias\", \"backbone.4.5.block.0.1.running_mean\", \"backbone.4.5.block.0.1.running_var\", \"backbone.4.5.block.0.1.num_batches_tracked\", \"backbone.4.5.block.1.0.weight\", \"backbone.4.5.block.1.1.weight\", \"backbone.4.5.block.1.1.bias\", \"backbone.4.5.block.1.1.running_mean\", \"backbone.4.5.block.1.1.running_var\", \"backbone.4.5.block.1.1.num_batches_tracked\", \"backbone.4.5.block.2.fc1.weight\", \"backbone.4.5.block.2.fc1.bias\", \"backbone.4.5.block.2.fc2.weight\", \"backbone.4.5.block.2.fc2.bias\", \"backbone.4.5.block.3.0.weight\", \"backbone.4.5.block.3.1.weight\", \"backbone.4.5.block.3.1.bias\", \"backbone.4.5.block.3.1.running_mean\", \"backbone.4.5.block.3.1.running_var\", \"backbone.4.5.block.3.1.num_batches_tracked\", \"backbone.4.0.block.0.0.weight\", \"backbone.4.0.block.0.1.weight\", \"backbone.4.0.block.0.1.bias\", \"backbone.4.0.block.0.1.running_mean\", \"backbone.4.0.block.0.1.running_var\", \"backbone.4.0.block.0.1.num_batches_tracked\", \"backbone.4.0.block.1.0.weight\", \"backbone.4.0.block.1.1.weight\", \"backbone.4.0.block.1.1.bias\", \"backbone.4.0.block.1.1.running_mean\", \"backbone.4.0.block.1.1.running_var\", \"backbone.4.0.block.1.1.num_batches_tracked\", \"backbone.4.0.block.2.fc1.weight\", \"backbone.4.0.block.2.fc1.bias\", \"backbone.4.0.block.2.fc2.weight\", \"backbone.4.0.block.2.fc2.bias\", \"backbone.4.0.block.3.0.weight\", \"backbone.4.0.block.3.1.weight\", \"backbone.4.0.block.3.1.bias\", \"backbone.4.0.block.3.1.running_mean\", \"backbone.4.0.block.3.1.running_var\", \"backbone.4.0.block.3.1.num_batches_tracked\", \"backbone.4.1.block.0.0.weight\", \"backbone.4.1.block.0.1.weight\", \"backbone.4.1.block.0.1.bias\", \"backbone.4.1.block.0.1.running_mean\", \"backbone.4.1.block.0.1.running_var\", \"backbone.4.1.block.0.1.num_batches_tracked\", \"backbone.4.1.block.1.0.weight\", \"backbone.4.1.block.1.1.weight\", \"backbone.4.1.block.1.1.bias\", \"backbone.4.1.block.1.1.running_mean\", \"backbone.4.1.block.1.1.running_var\", \"backbone.4.1.block.1.1.num_batches_tracked\", \"backbone.4.1.block.2.fc1.weight\", \"backbone.4.1.block.2.fc1.bias\", \"backbone.4.1.block.2.fc2.weight\", \"backbone.4.1.block.2.fc2.bias\", \"backbone.4.1.block.3.0.weight\", \"backbone.4.1.block.3.1.weight\", \"backbone.4.1.block.3.1.bias\", \"backbone.4.1.block.3.1.running_mean\", \"backbone.4.1.block.3.1.running_var\", \"backbone.4.1.block.3.1.num_batches_tracked\", \"backbone.5.2.block.0.0.weight\", \"backbone.5.2.block.0.1.weight\", \"backbone.5.2.block.0.1.bias\", \"backbone.5.2.block.0.1.running_mean\", \"backbone.5.2.block.0.1.running_var\", \"backbone.5.2.block.0.1.num_batches_tracked\", \"backbone.5.2.block.1.0.weight\", \"backbone.5.2.block.1.1.weight\", \"backbone.5.2.block.1.1.bias\", \"backbone.5.2.block.1.1.running_mean\", \"backbone.5.2.block.1.1.running_var\", \"backbone.5.2.block.1.1.num_batches_tracked\", \"backbone.5.2.block.2.fc1.weight\", \"backbone.5.2.block.2.fc1.bias\", \"backbone.5.2.block.2.fc2.weight\", \"backbone.5.2.block.2.fc2.bias\", \"backbone.5.2.block.3.0.weight\", \"backbone.5.2.block.3.1.weight\", \"backbone.5.2.block.3.1.bias\", \"backbone.5.2.block.3.1.running_mean\", \"backbone.5.2.block.3.1.running_var\", \"backbone.5.2.block.3.1.num_batches_tracked\", \"backbone.5.3.block.0.0.weight\", \"backbone.5.3.block.0.1.weight\", \"backbone.5.3.block.0.1.bias\", \"backbone.5.3.block.0.1.running_mean\", \"backbone.5.3.block.0.1.running_var\", \"backbone.5.3.block.0.1.num_batches_tracked\", \"backbone.5.3.block.1.0.weight\", \"backbone.5.3.block.1.1.weight\", \"backbone.5.3.block.1.1.bias\", \"backbone.5.3.block.1.1.running_mean\", \"backbone.5.3.block.1.1.running_var\", \"backbone.5.3.block.1.1.num_batches_tracked\", \"backbone.5.3.block.2.fc1.weight\", \"backbone.5.3.block.2.fc1.bias\", \"backbone.5.3.block.2.fc2.weight\", \"backbone.5.3.block.2.fc2.bias\", \"backbone.5.3.block.3.0.weight\", \"backbone.5.3.block.3.1.weight\", \"backbone.5.3.block.3.1.bias\", \"backbone.5.3.block.3.1.running_mean\", \"backbone.5.3.block.3.1.running_var\", \"backbone.5.3.block.3.1.num_batches_tracked\", \"backbone.5.4.block.0.0.weight\", \"backbone.5.4.block.0.1.weight\", \"backbone.5.4.block.0.1.bias\", \"backbone.5.4.block.0.1.running_mean\", \"backbone.5.4.block.0.1.running_var\", \"backbone.5.4.block.0.1.num_batches_tracked\", \"backbone.5.4.block.1.0.weight\", \"backbone.5.4.block.1.1.weight\", \"backbone.5.4.block.1.1.bias\", \"backbone.5.4.block.1.1.running_mean\", \"backbone.5.4.block.1.1.running_var\", \"backbone.5.4.block.1.1.num_batches_tracked\", \"backbone.5.4.block.2.fc1.weight\", \"backbone.5.4.block.2.fc1.bias\", \"backbone.5.4.block.2.fc2.weight\", \"backbone.5.4.block.2.fc2.bias\", \"backbone.5.4.block.3.0.weight\", \"backbone.5.4.block.3.1.weight\", \"backbone.5.4.block.3.1.bias\", \"backbone.5.4.block.3.1.running_mean\", \"backbone.5.4.block.3.1.running_var\", \"backbone.5.4.block.3.1.num_batches_tracked\", \"backbone.5.5.block.0.0.weight\", \"backbone.5.5.block.0.1.weight\", \"backbone.5.5.block.0.1.bias\", \"backbone.5.5.block.0.1.running_mean\", \"backbone.5.5.block.0.1.running_var\", \"backbone.5.5.block.0.1.num_batches_tracked\", \"backbone.5.5.block.1.0.weight\", \"backbone.5.5.block.1.1.weight\", \"backbone.5.5.block.1.1.bias\", \"backbone.5.5.block.1.1.running_mean\", \"backbone.5.5.block.1.1.running_var\", \"backbone.5.5.block.1.1.num_batches_tracked\", \"backbone.5.5.block.2.fc1.weight\", \"backbone.5.5.block.2.fc1.bias\", \"backbone.5.5.block.2.fc2.weight\", \"backbone.5.5.block.2.fc2.bias\", \"backbone.5.5.block.3.0.weight\", \"backbone.5.5.block.3.1.weight\", \"backbone.5.5.block.3.1.bias\", \"backbone.5.5.block.3.1.running_mean\", \"backbone.5.5.block.3.1.running_var\", \"backbone.5.5.block.3.1.num_batches_tracked\", \"backbone.5.6.block.0.0.weight\", \"backbone.5.6.block.0.1.weight\", \"backbone.5.6.block.0.1.bias\", \"backbone.5.6.block.0.1.running_mean\", \"backbone.5.6.block.0.1.running_var\", \"backbone.5.6.block.0.1.num_batches_tracked\", \"backbone.5.6.block.1.0.weight\", \"backbone.5.6.block.1.1.weight\", \"backbone.5.6.block.1.1.bias\", \"backbone.5.6.block.1.1.running_mean\", \"backbone.5.6.block.1.1.running_var\", \"backbone.5.6.block.1.1.num_batches_tracked\", \"backbone.5.6.block.2.fc1.weight\", \"backbone.5.6.block.2.fc1.bias\", \"backbone.5.6.block.2.fc2.weight\", \"backbone.5.6.block.2.fc2.bias\", \"backbone.5.6.block.3.0.weight\", \"backbone.5.6.block.3.1.weight\", \"backbone.5.6.block.3.1.bias\", \"backbone.5.6.block.3.1.running_mean\", \"backbone.5.6.block.3.1.running_var\", \"backbone.5.6.block.3.1.num_batches_tracked\", \"backbone.5.7.block.0.0.weight\", \"backbone.5.7.block.0.1.weight\", \"backbone.5.7.block.0.1.bias\", \"backbone.5.7.block.0.1.running_mean\", \"backbone.5.7.block.0.1.running_var\", \"backbone.5.7.block.0.1.num_batches_tracked\", \"backbone.5.7.block.1.0.weight\", \"backbone.5.7.block.1.1.weight\", \"backbone.5.7.block.1.1.bias\", \"backbone.5.7.block.1.1.running_mean\", \"backbone.5.7.block.1.1.running_var\", \"backbone.5.7.block.1.1.num_batches_tracked\", \"backbone.5.7.block.2.fc1.weight\", \"backbone.5.7.block.2.fc1.bias\", \"backbone.5.7.block.2.fc2.weight\", \"backbone.5.7.block.2.fc2.bias\", \"backbone.5.7.block.3.0.weight\", \"backbone.5.7.block.3.1.weight\", \"backbone.5.7.block.3.1.bias\", \"backbone.5.7.block.3.1.running_mean\", \"backbone.5.7.block.3.1.running_var\", \"backbone.5.7.block.3.1.num_batches_tracked\", \"backbone.5.8.block.0.0.weight\", \"backbone.5.8.block.0.1.weight\", \"backbone.5.8.block.0.1.bias\", \"backbone.5.8.block.0.1.running_mean\", \"backbone.5.8.block.0.1.running_var\", \"backbone.5.8.block.0.1.num_batches_tracked\", \"backbone.5.8.block.1.0.weight\", \"backbone.5.8.block.1.1.weight\", \"backbone.5.8.block.1.1.bias\", \"backbone.5.8.block.1.1.running_mean\", \"backbone.5.8.block.1.1.running_var\", \"backbone.5.8.block.1.1.num_batches_tracked\", \"backbone.5.8.block.2.fc1.weight\", \"backbone.5.8.block.2.fc1.bias\", \"backbone.5.8.block.2.fc2.weight\", \"backbone.5.8.block.2.fc2.bias\", \"backbone.5.8.block.3.0.weight\", \"backbone.5.8.block.3.1.weight\", \"backbone.5.8.block.3.1.bias\", \"backbone.5.8.block.3.1.running_mean\", \"backbone.5.8.block.3.1.running_var\", \"backbone.5.8.block.3.1.num_batches_tracked\", \"backbone.5.0.block.0.0.weight\", \"backbone.5.0.block.0.1.weight\", \"backbone.5.0.block.0.1.bias\", \"backbone.5.0.block.0.1.running_mean\", \"backbone.5.0.block.0.1.running_var\", \"backbone.5.0.block.0.1.num_batches_tracked\", \"backbone.5.0.block.1.0.weight\", \"backbone.5.0.block.1.1.weight\", \"backbone.5.0.block.1.1.bias\", \"backbone.5.0.block.1.1.running_mean\", \"backbone.5.0.block.1.1.running_var\", \"backbone.5.0.block.1.1.num_batches_tracked\", \"backbone.5.0.block.2.fc1.weight\", \"backbone.5.0.block.2.fc1.bias\", \"backbone.5.0.block.2.fc2.weight\", \"backbone.5.0.block.2.fc2.bias\", \"backbone.5.0.block.3.0.weight\", \"backbone.5.0.block.3.1.weight\", \"backbone.5.0.block.3.1.bias\", \"backbone.5.0.block.3.1.running_mean\", \"backbone.5.0.block.3.1.running_var\", \"backbone.5.0.block.3.1.num_batches_tracked\", \"backbone.5.1.block.0.0.weight\", \"backbone.5.1.block.0.1.weight\", \"backbone.5.1.block.0.1.bias\", \"backbone.5.1.block.0.1.running_mean\", \"backbone.5.1.block.0.1.running_var\", \"backbone.5.1.block.0.1.num_batches_tracked\", \"backbone.5.1.block.1.0.weight\", \"backbone.5.1.block.1.1.weight\", \"backbone.5.1.block.1.1.bias\", \"backbone.5.1.block.1.1.running_mean\", \"backbone.5.1.block.1.1.running_var\", \"backbone.5.1.block.1.1.num_batches_tracked\", \"backbone.5.1.block.2.fc1.weight\", \"backbone.5.1.block.2.fc1.bias\", \"backbone.5.1.block.2.fc2.weight\", \"backbone.5.1.block.2.fc2.bias\", \"backbone.5.1.block.3.0.weight\", \"backbone.5.1.block.3.1.weight\", \"backbone.5.1.block.3.1.bias\", \"backbone.5.1.block.3.1.running_mean\", \"backbone.5.1.block.3.1.running_var\", \"backbone.5.1.block.3.1.num_batches_tracked\", \"backbone.6.2.block.0.0.weight\", \"backbone.6.2.block.0.1.weight\", \"backbone.6.2.block.0.1.bias\", \"backbone.6.2.block.0.1.running_mean\", \"backbone.6.2.block.0.1.running_var\", \"backbone.6.2.block.0.1.num_batches_tracked\", \"backbone.6.2.block.1.0.weight\", \"backbone.6.2.block.1.1.weight\", \"backbone.6.2.block.1.1.bias\", \"backbone.6.2.block.1.1.running_mean\", \"backbone.6.2.block.1.1.running_var\", \"backbone.6.2.block.1.1.num_batches_tracked\", \"backbone.6.2.block.2.fc1.weight\", \"backbone.6.2.block.2.fc1.bias\", \"backbone.6.2.block.2.fc2.weight\", \"backbone.6.2.block.2.fc2.bias\", \"backbone.6.2.block.3.0.weight\", \"backbone.6.2.block.3.1.weight\", \"backbone.6.2.block.3.1.bias\", \"backbone.6.2.block.3.1.running_mean\", \"backbone.6.2.block.3.1.running_var\", \"backbone.6.2.block.3.1.num_batches_tracked\", \"backbone.6.3.block.0.0.weight\", \"backbone.6.3.block.0.1.weight\", \"backbone.6.3.block.0.1.bias\", \"backbone.6.3.block.0.1.running_mean\", \"backbone.6.3.block.0.1.running_var\", \"backbone.6.3.block.0.1.num_batches_tracked\", \"backbone.6.3.block.1.0.weight\", \"backbone.6.3.block.1.1.weight\", \"backbone.6.3.block.1.1.bias\", \"backbone.6.3.block.1.1.running_mean\", \"backbone.6.3.block.1.1.running_var\", \"backbone.6.3.block.1.1.num_batches_tracked\", \"backbone.6.3.block.2.fc1.weight\", \"backbone.6.3.block.2.fc1.bias\", \"backbone.6.3.block.2.fc2.weight\", \"backbone.6.3.block.2.fc2.bias\", \"backbone.6.3.block.3.0.weight\", \"backbone.6.3.block.3.1.weight\", \"backbone.6.3.block.3.1.bias\", \"backbone.6.3.block.3.1.running_mean\", \"backbone.6.3.block.3.1.running_var\", \"backbone.6.3.block.3.1.num_batches_tracked\", \"backbone.6.4.block.0.0.weight\", \"backbone.6.4.block.0.1.weight\", \"backbone.6.4.block.0.1.bias\", \"backbone.6.4.block.0.1.running_mean\", \"backbone.6.4.block.0.1.running_var\", \"backbone.6.4.block.0.1.num_batches_tracked\", \"backbone.6.4.block.1.0.weight\", \"backbone.6.4.block.1.1.weight\", \"backbone.6.4.block.1.1.bias\", \"backbone.6.4.block.1.1.running_mean\", \"backbone.6.4.block.1.1.running_var\", \"backbone.6.4.block.1.1.num_batches_tracked\", \"backbone.6.4.block.2.fc1.weight\", \"backbone.6.4.block.2.fc1.bias\", \"backbone.6.4.block.2.fc2.weight\", \"backbone.6.4.block.2.fc2.bias\", \"backbone.6.4.block.3.0.weight\", \"backbone.6.4.block.3.1.weight\", \"backbone.6.4.block.3.1.bias\", \"backbone.6.4.block.3.1.running_mean\", \"backbone.6.4.block.3.1.running_var\", \"backbone.6.4.block.3.1.num_batches_tracked\", \"backbone.6.5.block.0.0.weight\", \"backbone.6.5.block.0.1.weight\", \"backbone.6.5.block.0.1.bias\", \"backbone.6.5.block.0.1.running_mean\", \"backbone.6.5.block.0.1.running_var\", \"backbone.6.5.block.0.1.num_batches_tracked\", \"backbone.6.5.block.1.0.weight\", \"backbone.6.5.block.1.1.weight\", \"backbone.6.5.block.1.1.bias\", \"backbone.6.5.block.1.1.running_mean\", \"backbone.6.5.block.1.1.running_var\", \"backbone.6.5.block.1.1.num_batches_tracked\", \"backbone.6.5.block.2.fc1.weight\", \"backbone.6.5.block.2.fc1.bias\", \"backbone.6.5.block.2.fc2.weight\", \"backbone.6.5.block.2.fc2.bias\", \"backbone.6.5.block.3.0.weight\", \"backbone.6.5.block.3.1.weight\", \"backbone.6.5.block.3.1.bias\", \"backbone.6.5.block.3.1.running_mean\", \"backbone.6.5.block.3.1.running_var\", \"backbone.6.5.block.3.1.num_batches_tracked\", \"backbone.6.6.block.0.0.weight\", \"backbone.6.6.block.0.1.weight\", \"backbone.6.6.block.0.1.bias\", \"backbone.6.6.block.0.1.running_mean\", \"backbone.6.6.block.0.1.running_var\", \"backbone.6.6.block.0.1.num_batches_tracked\", \"backbone.6.6.block.1.0.weight\", \"backbone.6.6.block.1.1.weight\", \"backbone.6.6.block.1.1.bias\", \"backbone.6.6.block.1.1.running_mean\", \"backbone.6.6.block.1.1.running_var\", \"backbone.6.6.block.1.1.num_batches_tracked\", \"backbone.6.6.block.2.fc1.weight\", \"backbone.6.6.block.2.fc1.bias\", \"backbone.6.6.block.2.fc2.weight\", \"backbone.6.6.block.2.fc2.bias\", \"backbone.6.6.block.3.0.weight\", \"backbone.6.6.block.3.1.weight\", \"backbone.6.6.block.3.1.bias\", \"backbone.6.6.block.3.1.running_mean\", \"backbone.6.6.block.3.1.running_var\", \"backbone.6.6.block.3.1.num_batches_tracked\", \"backbone.6.7.block.0.0.weight\", \"backbone.6.7.block.0.1.weight\", \"backbone.6.7.block.0.1.bias\", \"backbone.6.7.block.0.1.running_mean\", \"backbone.6.7.block.0.1.running_var\", \"backbone.6.7.block.0.1.num_batches_tracked\", \"backbone.6.7.block.1.0.weight\", \"backbone.6.7.block.1.1.weight\", \"backbone.6.7.block.1.1.bias\", \"backbone.6.7.block.1.1.running_mean\", \"backbone.6.7.block.1.1.running_var\", \"backbone.6.7.block.1.1.num_batches_tracked\", \"backbone.6.7.block.2.fc1.weight\", \"backbone.6.7.block.2.fc1.bias\", \"backbone.6.7.block.2.fc2.weight\", \"backbone.6.7.block.2.fc2.bias\", \"backbone.6.7.block.3.0.weight\", \"backbone.6.7.block.3.1.weight\", \"backbone.6.7.block.3.1.bias\", \"backbone.6.7.block.3.1.running_mean\", \"backbone.6.7.block.3.1.running_var\", \"backbone.6.7.block.3.1.num_batches_tracked\", \"backbone.6.8.block.0.0.weight\", \"backbone.6.8.block.0.1.weight\", \"backbone.6.8.block.0.1.bias\", \"backbone.6.8.block.0.1.running_mean\", \"backbone.6.8.block.0.1.running_var\", \"backbone.6.8.block.0.1.num_batches_tracked\", \"backbone.6.8.block.1.0.weight\", \"backbone.6.8.block.1.1.weight\", \"backbone.6.8.block.1.1.bias\", \"backbone.6.8.block.1.1.running_mean\", \"backbone.6.8.block.1.1.running_var\", \"backbone.6.8.block.1.1.num_batches_tracked\", \"backbone.6.8.block.2.fc1.weight\", \"backbone.6.8.block.2.fc1.bias\", \"backbone.6.8.block.2.fc2.weight\", \"backbone.6.8.block.2.fc2.bias\", \"backbone.6.8.block.3.0.weight\", \"backbone.6.8.block.3.1.weight\", \"backbone.6.8.block.3.1.bias\", \"backbone.6.8.block.3.1.running_mean\", \"backbone.6.8.block.3.1.running_var\", \"backbone.6.8.block.3.1.num_batches_tracked\", \"backbone.6.9.block.0.0.weight\", \"backbone.6.9.block.0.1.weight\", \"backbone.6.9.block.0.1.bias\", \"backbone.6.9.block.0.1.running_mean\", \"backbone.6.9.block.0.1.running_var\", \"backbone.6.9.block.0.1.num_batches_tracked\", \"backbone.6.9.block.1.0.weight\", \"backbone.6.9.block.1.1.weight\", \"backbone.6.9.block.1.1.bias\", \"backbone.6.9.block.1.1.running_mean\", \"backbone.6.9.block.1.1.running_var\", \"backbone.6.9.block.1.1.num_batches_tracked\", \"backbone.6.9.block.2.fc1.weight\", \"backbone.6.9.block.2.fc1.bias\", \"backbone.6.9.block.2.fc2.weight\", \"backbone.6.9.block.2.fc2.bias\", \"backbone.6.9.block.3.0.weight\", \"backbone.6.9.block.3.1.weight\", \"backbone.6.9.block.3.1.bias\", \"backbone.6.9.block.3.1.running_mean\", \"backbone.6.9.block.3.1.running_var\", \"backbone.6.9.block.3.1.num_batches_tracked\", \"backbone.6.10.block.0.0.weight\", \"backbone.6.10.block.0.1.weight\", \"backbone.6.10.block.0.1.bias\", \"backbone.6.10.block.0.1.running_mean\", \"backbone.6.10.block.0.1.running_var\", \"backbone.6.10.block.0.1.num_batches_tracked\", \"backbone.6.10.block.1.0.weight\", \"backbone.6.10.block.1.1.weight\", \"backbone.6.10.block.1.1.bias\", \"backbone.6.10.block.1.1.running_mean\", \"backbone.6.10.block.1.1.running_var\", \"backbone.6.10.block.1.1.num_batches_tracked\", \"backbone.6.10.block.2.fc1.weight\", \"backbone.6.10.block.2.fc1.bias\", \"backbone.6.10.block.2.fc2.weight\", \"backbone.6.10.block.2.fc2.bias\", \"backbone.6.10.block.3.0.weight\", \"backbone.6.10.block.3.1.weight\", \"backbone.6.10.block.3.1.bias\", \"backbone.6.10.block.3.1.running_mean\", \"backbone.6.10.block.3.1.running_var\", \"backbone.6.10.block.3.1.num_batches_tracked\", \"backbone.6.11.block.0.0.weight\", \"backbone.6.11.block.0.1.weight\", \"backbone.6.11.block.0.1.bias\", \"backbone.6.11.block.0.1.running_mean\", \"backbone.6.11.block.0.1.running_var\", \"backbone.6.11.block.0.1.num_batches_tracked\", \"backbone.6.11.block.1.0.weight\", \"backbone.6.11.block.1.1.weight\", \"backbone.6.11.block.1.1.bias\", \"backbone.6.11.block.1.1.running_mean\", \"backbone.6.11.block.1.1.running_var\", \"backbone.6.11.block.1.1.num_batches_tracked\", \"backbone.6.11.block.2.fc1.weight\", \"backbone.6.11.block.2.fc1.bias\", \"backbone.6.11.block.2.fc2.weight\", \"backbone.6.11.block.2.fc2.bias\", \"backbone.6.11.block.3.0.weight\", \"backbone.6.11.block.3.1.weight\", \"backbone.6.11.block.3.1.bias\", \"backbone.6.11.block.3.1.running_mean\", \"backbone.6.11.block.3.1.running_var\", \"backbone.6.11.block.3.1.num_batches_tracked\", \"backbone.6.12.block.0.0.weight\", \"backbone.6.12.block.0.1.weight\", \"backbone.6.12.block.0.1.bias\", \"backbone.6.12.block.0.1.running_mean\", \"backbone.6.12.block.0.1.running_var\", \"backbone.6.12.block.0.1.num_batches_tracked\", \"backbone.6.12.block.1.0.weight\", \"backbone.6.12.block.1.1.weight\", \"backbone.6.12.block.1.1.bias\", \"backbone.6.12.block.1.1.running_mean\", \"backbone.6.12.block.1.1.running_var\", \"backbone.6.12.block.1.1.num_batches_tracked\", \"backbone.6.12.block.2.fc1.weight\", \"backbone.6.12.block.2.fc1.bias\", \"backbone.6.12.block.2.fc2.weight\", \"backbone.6.12.block.2.fc2.bias\", \"backbone.6.12.block.3.0.weight\", \"backbone.6.12.block.3.1.weight\", \"backbone.6.12.block.3.1.bias\", \"backbone.6.12.block.3.1.running_mean\", \"backbone.6.12.block.3.1.running_var\", \"backbone.6.12.block.3.1.num_batches_tracked\", \"backbone.6.13.block.0.0.weight\", \"backbone.6.13.block.0.1.weight\", \"backbone.6.13.block.0.1.bias\", \"backbone.6.13.block.0.1.running_mean\", \"backbone.6.13.block.0.1.running_var\", \"backbone.6.13.block.0.1.num_batches_tracked\", \"backbone.6.13.block.1.0.weight\", \"backbone.6.13.block.1.1.weight\", \"backbone.6.13.block.1.1.bias\", \"backbone.6.13.block.1.1.running_mean\", \"backbone.6.13.block.1.1.running_var\", \"backbone.6.13.block.1.1.num_batches_tracked\", \"backbone.6.13.block.2.fc1.weight\", \"backbone.6.13.block.2.fc1.bias\", \"backbone.6.13.block.2.fc2.weight\", \"backbone.6.13.block.2.fc2.bias\", \"backbone.6.13.block.3.0.weight\", \"backbone.6.13.block.3.1.weight\", \"backbone.6.13.block.3.1.bias\", \"backbone.6.13.block.3.1.running_mean\", \"backbone.6.13.block.3.1.running_var\", \"backbone.6.13.block.3.1.num_batches_tracked\", \"backbone.6.14.block.0.0.weight\", \"backbone.6.14.block.0.1.weight\", \"backbone.6.14.block.0.1.bias\", \"backbone.6.14.block.0.1.running_mean\", \"backbone.6.14.block.0.1.running_var\", \"backbone.6.14.block.0.1.num_batches_tracked\", \"backbone.6.14.block.1.0.weight\", \"backbone.6.14.block.1.1.weight\", \"backbone.6.14.block.1.1.bias\", \"backbone.6.14.block.1.1.running_mean\", \"backbone.6.14.block.1.1.running_var\", \"backbone.6.14.block.1.1.num_batches_tracked\", \"backbone.6.14.block.2.fc1.weight\", \"backbone.6.14.block.2.fc1.bias\", \"backbone.6.14.block.2.fc2.weight\", \"backbone.6.14.block.2.fc2.bias\", \"backbone.6.14.block.3.0.weight\", \"backbone.6.14.block.3.1.weight\", \"backbone.6.14.block.3.1.bias\", \"backbone.6.14.block.3.1.running_mean\", \"backbone.6.14.block.3.1.running_var\", \"backbone.6.14.block.3.1.num_batches_tracked\", \"backbone.6.0.block.0.0.weight\", \"backbone.6.0.block.0.1.weight\", \"backbone.6.0.block.0.1.bias\", \"backbone.6.0.block.0.1.running_mean\", \"backbone.6.0.block.0.1.running_var\", \"backbone.6.0.block.0.1.num_batches_tracked\", \"backbone.6.0.block.1.0.weight\", \"backbone.6.0.block.1.1.weight\", \"backbone.6.0.block.1.1.bias\", \"backbone.6.0.block.1.1.running_mean\", \"backbone.6.0.block.1.1.running_var\", \"backbone.6.0.block.1.1.num_batches_tracked\", \"backbone.6.0.block.2.fc1.weight\", \"backbone.6.0.block.2.fc1.bias\", \"backbone.6.0.block.2.fc2.weight\", \"backbone.6.0.block.2.fc2.bias\", \"backbone.6.0.block.3.0.weight\", \"backbone.6.0.block.3.1.weight\", \"backbone.6.0.block.3.1.bias\", \"backbone.6.0.block.3.1.running_mean\", \"backbone.6.0.block.3.1.running_var\", \"backbone.6.0.block.3.1.num_batches_tracked\", \"backbone.6.1.block.0.0.weight\", \"backbone.6.1.block.0.1.weight\", \"backbone.6.1.block.0.1.bias\", \"backbone.6.1.block.0.1.running_mean\", \"backbone.6.1.block.0.1.running_var\", \"backbone.6.1.block.0.1.num_batches_tracked\", \"backbone.6.1.block.1.0.weight\", \"backbone.6.1.block.1.1.weight\", \"backbone.6.1.block.1.1.bias\", \"backbone.6.1.block.1.1.running_mean\", \"backbone.6.1.block.1.1.running_var\", \"backbone.6.1.block.1.1.num_batches_tracked\", \"backbone.6.1.block.2.fc1.weight\", \"backbone.6.1.block.2.fc1.bias\", \"backbone.6.1.block.2.fc2.weight\", \"backbone.6.1.block.2.fc2.bias\", \"backbone.6.1.block.3.0.weight\", \"backbone.6.1.block.3.1.weight\", \"backbone.6.1.block.3.1.bias\", \"backbone.6.1.block.3.1.running_mean\", \"backbone.6.1.block.3.1.running_var\", \"backbone.6.1.block.3.1.num_batches_tracked\", \"backbone.7.0.weight\", \"backbone.7.1.weight\", \"backbone.7.1.bias\", \"backbone.7.1.running_mean\", \"backbone.7.1.running_var\", \"backbone.7.1.num_batches_tracked\". \n",
            "\tsize mismatch for aggregation.3.weight: copying a param with shape torch.Size([512, 1280]) from checkpoint, the shape in current model is torch.Size([512, 512]).\n",
            "\n",
            "2023-02-05 06:36:32   Experiment finished (with some errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGRL.py --dataset_folder /content/small/ --resume_model /content/logs/default/2023-02-05_06-18-50/best_model.pth --grl_param 0.3 #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "P30k-Qy-XuGX"
      },
      "source": [
        "# Step 8: Ensemble"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "aQeE9OUhXwSu"
      },
      "source": [
        "To improve the recalls we try to concatenate the descriptors obtained with different models (ensemble)."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "KhGDUXS4YXb1"
      },
      "source": [
        "Before testing, check that there are these files:\n",
        "- model_ensembler.py (in network)\n",
        "- modify network.py\n",
        "- evalEnsembler.py\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZW7h08MQXwtE",
        "outputId": "208ba5fc-f960-4356-946d-840161e561f0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "fatal: destination path 'CosPlace' already exists and is not an empty directory.\n"
          ]
        }
      ],
      "source": [
        "# download code of CosPlace\n",
        "!git clone \"https://github.com/gmberton/CosPlace\" \n",
        "#!rm -r \"/content/CosPlace\""
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "S4TA_2R_b3eH"
      },
      "source": [
        "Might be useless"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YhM5dSr4b29A"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import sys\n",
        "import torch\n",
        "import logging\n",
        "import multiprocessing\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "from datetime import datetime\n",
        "\n",
        "sys.path.append(\"/content/CosPlace/\")\n",
        "import CosPlace\n",
        "from CosPlace import *\n",
        "\n",
        "torch.backends.cudnn.benchmark = True  # Provides a speedup"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "w8PkoLFC7as9"
      },
      "source": [
        "## Method 1 (model soup)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "yVJMyrtMBT3d"
      },
      "source": [
        "### sf-xs "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "buYCvFAiBXpY"
      },
      "source": [
        "Results 03/02/2023 \n",
        "\n",
        "Potential greedy soup val acc 52.0, best so far 51.0.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y4f8AH5OJ0CV"
      },
      "outputs": [],
      "source": [
        "!cp -r /content/tokyo_xs/test/database /content/tokyo-night"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6-x5M-R2cHUo",
        "outputId": "e35855b7-fe5a-4253-8656-855e9c773d41"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/content/evalEnsembler.py\", line 75, in <module>\n",
            "    parser = parser.ArgumentParser()\n",
            "AttributeError: module 'parser' has no attribute 'ArgumentParser'\n"
          ]
        }
      ],
      "source": [
        "#Modificare cartella in base al test\n",
        "!python /content/evalEnsembler.py --dataset_folder /content/small/"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "2jmEvT4WBcU6"
      },
      "source": [
        "### Tokyo xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "IX24ZSzaBd8U"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "Potential greedy soup val acc 67.3015873015873, best so far 68.25396825396825."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MVq8rPQn4L5s",
        "outputId": "12bc5da3-c802-43f1-f039-f496d04b60fd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:02<00:00,  6.50it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:06<00:00, 47.02it/s]\n",
            "Potential greedy soup val acc 68.25396825396825, best so far 1.2.\n",
            "Adding to soup.\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [01:56<00:00,  6.84it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:04<00:00, 68.26it/s]\n",
            "Potential greedy soup val acc 67.3015873015873, best so far 68.25396825396825.\n"
          ]
        }
      ],
      "source": [
        "#Modificare cartella in base al test\n",
        "!python /content/evalEnsembler.py --dataset_folder /content/tokyo_xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "4d1GXiRABjy3"
      },
      "source": [
        "### Tokyo night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "vlktjQwNBlkC"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "Potential greedy soup val acc 47.61904761904761, best so far 45.714285714285715."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dzzd3NgO5Ub3",
        "outputId": "ac529f7a-388d-4ee7-81a2-c00ab3adf335"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:03<00:00,  6.45it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 49.28it/s]\n",
            "Potential greedy soup val acc 45.714285714285715, best so far 1.2.\n",
            "Adding to soup.\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [01:57<00:00,  6.79it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 36.33it/s]\n",
            "Potential greedy soup val acc 47.61904761904761, best so far 45.714285714285715.\n",
            "Adding to soup.\n"
          ]
        }
      ],
      "source": [
        "#Modificare cartella in base al test\n",
        "!python /content/evalEnsembler.py --dataset_folder /content/tokyo-night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "twPkItbR7dJw"
      },
      "source": [
        "## Method 2 (ensemble)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ZPrn9m_57gBN"
      },
      "source": [
        "- evalensemble2.py needs to be added\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4UCuF16hAeUy"
      },
      "outputs": [],
      "source": [
        "# download code of CosPlace\n",
        "!git clone \"https://github.com/gmberton/CosPlace\" \n",
        "#!rm -r \"/content/CosPlace\""
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "UYrQa4avB5av"
      },
      "source": [
        "### sf-xs, tokyo xs, tokyo night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "qC3OI4JIB7oQ"
      },
      "source": [
        "Results 03/02/2023\n",
        "\n",
        "sf-xs: R@1: 35.6, R@5: 47.7, R@10: 54.1, R@20: 59.0\n",
        "\n",
        "tokyo xs: R@1: 43.2, R@5: 61.9, R@10: 68.9, R@20: 78.7\n",
        "\n",
        "tokyo night: R@1: 35.2, R@5: 53.3, R@10: 62.9, R@20: 68.6\n",
        "\n",
        "Arcface+cosplace+sphereface"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "972VMxA0CIwx",
        "outputId": "0515307f-637b-4c27-cddc-553d680da739"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "\r  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:55<00:00,  4.09it/s]\n",
            "  0%|                                                                      | 0/1000 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:35<00:00, 28.32it/s]\n",
            "R@1: 35.6, R@5: 47.7, R@10: 54.1, R@20: 59.0\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:51<00:00,  3.46it/s]\n",
            "  0%|                                                                       | 0/315 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:09<00:00, 33.32it/s]\n",
            "R@1: 43.2, R@5: 61.9, R@10: 68.9, R@20: 78.7\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:54<00:00,  3.41it/s]\n",
            "  0%|                                                                       | 0/105 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 29.57it/s]\n",
            "R@1: 35.2, R@5: 53.3, R@10: 62.9, R@20: 68.6\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalensemble2.py --dataset_folder /content/small"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e-YUZ-zqUflM",
        "outputId": "2f416c7c-e50f-45de-c7e0-db85cd1c9604"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "\r  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [07:01<00:00,  4.03it/s]\n",
            "  0%|                                                                      | 0/1000 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:35<00:00, 28.15it/s]\n",
            "R@1: 35.6, R@5: 47.7, R@10: 54.1, R@20: 59.0\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:49<00:00,  3.48it/s]\n",
            "  0%|                                                                       | 0/315 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 35.65it/s]\n",
            "R@1: 43.2, R@5: 61.9, R@10: 68.9, R@20: 78.7\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:54<00:00,  3.41it/s]\n",
            "  0%|                                                                       | 0/105 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:05<00:00, 20.80it/s]\n",
            "R@1: 35.2, R@5: 53.3, R@10: 62.9, R@20: 68.6\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalensemble2.py --dataset_folder /content/tokyo-night"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_Js1ygxoUf2U",
        "outputId": "dbf8e49d-e3ef-4353-e4c3-052788ffc996"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "\r  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:57<00:00,  4.07it/s]\n",
            "  0%|                                                                      | 0/1000 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:35<00:00, 28.39it/s]\n",
            "R@1: 35.6, R@5: 47.7, R@10: 54.1, R@20: 59.0\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:49<00:00,  3.48it/s]\n",
            "  0%|                                                                       | 0/315 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:11<00:00, 27.36it/s]\n",
            "R@1: 43.2, R@5: 61.9, R@10: 68.9, R@20: 78.7\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:49<00:00,  3.48it/s]\n",
            "  0%|                                                                       | 0/105 [00:00<?, ?it/s]/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:04<00:00, 26.12it/s]\n",
            "R@1: 35.2, R@5: 53.3, R@10: 62.9, R@20: 68.6\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalensemble2.py --dataset_folder /content/tokyo_xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "GKpEYf8hAz5C"
      },
      "source": [
        "# Step 9: Multi scale - AVG "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "N7EdPKcjA5AH"
      },
      "source": [
        "- test.py modified\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "\n",
        "import faiss\n",
        "import torch\n",
        "import logging\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "from typing import Tuple\n",
        "from argparse import Namespace\n",
        "from torch.utils.data.dataset import Subset\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "import torchvision.transforms as transforms\n",
        "from PIL import Image\n",
        "from dataset_warp import compute_warping\n",
        "\n",
        "\n",
        "# Compute R@1, R@5, R@10, R@20\n",
        "RECALL_VALUES = [1, 5, 10, 20]\n",
        "\n",
        "\n",
        "def test(args: Namespace, eval_ds: Dataset, model: torch.nn.Module) -> Tuple[np.ndarray, str]:\n",
        "    \"\"\"Compute descriptors of the given dataset and compute the recalls.\"\"\"\n",
        "    \n",
        "    model = model.eval()\n",
        "    if args.multi_scale:\n",
        "        logging.info(f\"Test with multi-scale, the multi-scale method is: {args.multi_scale_method}\")\n",
        "    with torch.no_grad():\n",
        "        logging.debug(\"Extracting database descriptors for evaluation/testing\")\n",
        "        database_subset_ds = Subset(eval_ds, list(range(eval_ds.database_num)))\n",
        "        database_dataloader = DataLoader(dataset=database_subset_ds, num_workers=args.num_workers,\n",
        "                                         batch_size=args.infer_batch_size, pin_memory=(args.device == \"cuda\"))\n",
        "        all_descriptors = np.empty((len(eval_ds), args.fc_output_dim), dtype=\"float32\")\n",
        "        for images, indices in tqdm(database_dataloader, ncols=100):\n",
        "            descriptors = model(images.to(args.device))\n",
        "            descriptors = descriptors.cpu().numpy()\n",
        "            all_descriptors[indices.numpy(), :] = descriptors\n",
        "        \n",
        "        logging.debug(\"Extracting queries descriptors for evaluation/testing using batch size 1\")\n",
        "        queries_infer_batch_size = 1\n",
        "        queries_subset_ds = Subset(eval_ds, list(range(eval_ds.database_num, eval_ds.database_num+eval_ds.queries_num)))\n",
        "        queries_dataloader = DataLoader(dataset=queries_subset_ds, num_workers=args.num_workers,\n",
        "                                        batch_size=queries_infer_batch_size, pin_memory=(args.device == \"cuda\"))\n",
        "        for images, indices in tqdm(queries_dataloader, ncols=100):\n",
        "            #descriptors = model(images.to(args.device))\n",
        "            \n",
        "            #\n",
        "            if args.multi_scale and args.multi_scale_method == 'avg':\n",
        "                H = args.resize[0]\n",
        "                W = args.resize[1]\n",
        "                HxW = args.resize   \n",
        "                original = images \n",
        "                # create the resolution lists \n",
        "                H_list = [int(H/i) for i in args.select_resolutions]             \n",
        "                W_list = [int(W/i) for i in args.select_resolutions]\n",
        "                \n",
        "                multi_scale = []\n",
        "                for i, j in zip(H_list, W_list):\n",
        "                    size = (i, j)                                                   # size resolution of the resize\n",
        "                    tra = torch.nn.Sequential(transforms.Resize(size))              # creating the transformation \n",
        "                    tra2 = torch.nn.Sequential(transforms.Resize(HxW))\n",
        "                    tmp_query = tra(original)                                       # transforming the img\n",
        "                    img = tra2(tmp_query)\n",
        "                    descriptors = model(img.to(args.device))\n",
        "                    multi_scale.append(descriptors)\n",
        "                feature = torch.stack(multi_scale, -1)\n",
        "                descriptors = torch.mean(feature.type(torch.float32), dim=-1)\n",
        "            \n",
        "            elif args.multi_scale and args.multi_scale_method == 'sum':\n",
        "                H = args.resize[0]\n",
        "                W = args.resize[1]\n",
        "                HxW = args.resize   \n",
        "                original = images \n",
        "                # create the resolution lists \n",
        "                H_list = [int(H/i) for i in args.select_resolutions]             \n",
        "                W_list = [int(W/i) for i in args.select_resolutions]\n",
        "                \n",
        "                multi_scale = []\n",
        "                for i, j in zip(H_list, W_list):\n",
        "                    size = (i, j)                                                   # size resolution of the resize\n",
        "                    tra = torch.nn.Sequential(transforms.Resize(size))              # creating the transformation \n",
        "                    tra2 = torch.nn.Sequential(transforms.Resize(HxW))\n",
        "                    tmp_query = tra(original)                                       # transforming the img\n",
        "                    img = tra2(tmp_query)\n",
        "                    feature_ms = model(img.to(args.device))\n",
        "                    multi_scale.append(feature_ms)\n",
        "                feature = torch.stack(multi_scale, -1)\n",
        "                descriptors = torch.sum(feature.type(torch.float32), dim=-1)\n",
        "            elif args.multi_scale and args.multi_scale_method == 'max':\n",
        "                H = args.resize[0]\n",
        "                W = args.resize[1]\n",
        "                HxW = args.resize   \n",
        "                original = images \n",
        "                # create the resolution lists \n",
        "                H_list = [int(H/i) for i in args.select_resolutions]             \n",
        "                W_list = [int(W/i) for i in args.select_resolutions]\n",
        "                \n",
        "                multi_scale = []\n",
        "                for i, j in zip(H_list, W_list):\n",
        "                    size = (i, j)                                                   # size resolution of the resize\n",
        "                    tra = torch.nn.Sequential(transforms.Resize(size))              # creating the transformation \n",
        "                    tra2 = torch.nn.Sequential(transforms.Resize(HxW))\n",
        "                    tmp_query = tra(original)                                       # transforming the img\n",
        "                    img = tra2(tmp_query)\n",
        "                    feature_ms = model(img.to(args.device))\n",
        "                    multi_scale.append(feature_ms)\n",
        "                feature = torch.stack(multi_scale, -1)\n",
        "                descriptors, max_index = torch.max(feature.type(torch.float32), dim=-1)\n",
        "                del max_index\n",
        "            elif args.multi_scale and args.multi_scale_method == 'min':\n",
        "                H = args.resize[0]\n",
        "                W = args.resize[1]\n",
        "                HxW = args.resize   \n",
        "                original = images\n",
        "                # create the resolution lists \n",
        "                H_list = [int(H/i) for i in args.select_resolutions]             \n",
        "                W_list = [int(W/i) for i in args.select_resolutions]\n",
        "                \n",
        "                multi_scale = []\n",
        "                for i, j in zip(H_list, W_list):\n",
        "                    size = (i, j)                                                   # size resolution of the resize\n",
        "                    tra = torch.nn.Sequential(transforms.Resize(size))              # creating the transformation \n",
        "                    tra2 = torch.nn.Sequential(transforms.Resize(HxW))\n",
        "                    tmp_query = tra(original)                                       # transforming the img\n",
        "                    img = tra2(tmp_query)\n",
        "                    feature_ms = model(img.to(args.device))\n",
        "                    multi_scale.append(feature_ms)\n",
        "                feature = torch.stack(multi_scale, -1)\n",
        "                descriptors, min_index = torch.min(feature.type(torch.float32), dim=-1)\n",
        "                del min_index\n",
        "            else:\n",
        "                descriptor = model(images.to(args.device))\n",
        "            descriptors = descriptors.cpu().numpy()\n",
        "            all_descriptors[indices.numpy(), :] = descriptors\n",
        "    queries_descriptors = all_descriptors[eval_ds.database_num:]\n",
        "    database_descriptors = all_descriptors[:eval_ds.database_num]\n",
        "    \n",
        "    # Use a kNN to find predictions\n",
        "    faiss_index = faiss.IndexFlatL2(args.fc_output_dim)\n",
        "    faiss_index.add(database_descriptors)\n",
        "    del database_descriptors, all_descriptors\n",
        "    \n",
        "    logging.debug(\"Calculating recalls\")\n",
        "    _, predictions = faiss_index.search(queries_descriptors, max(RECALL_VALUES))\n",
        "    \n",
        "    #### For each query, check if the predictions are correct\n",
        "    positives_per_query = eval_ds.get_positives()\n",
        "    recalls = np.zeros(len(RECALL_VALUES))\n",
        "    for query_index, preds in enumerate(predictions):\n",
        "        for i, n in enumerate(RECALL_VALUES):\n",
        "            if np.any(np.in1d(preds[:n], positives_per_query[query_index])):\n",
        "                recalls[i:] += 1\n",
        "                break\n",
        "    # Divide by queries_num and multiply by 100, so the recalls are in percentages\n",
        "    recalls = recalls / eval_ds.queries_num * 100\n",
        "    recalls_str = \", \".join([f\"R@{val}: {rec:.1f}\" for val, rec in zip(RECALL_VALUES, recalls)])\n",
        "    return recalls, recalls_str\n",
        "\n",
        "\n",
        "# test usato per calcolare recall e recall_str è uguale a quello sopra, solo che la rete è diversa quindi devo chiamre il model diversamente\n",
        "def use_geowarp(args: Namespace, eval_ds: Dataset, model: torch.nn.Module):\n",
        "    \"\"\"Compute descriptors of the given dataset and compute the recalls.\"\"\"\n",
        "    \n",
        "    model = model.eval()                                                        # si mette il modello in evaluation mode\n",
        "    with torch.no_grad():                                                       # all'interno del ciclo, il gradient è disabilitato (requires_grad=False)\n",
        "        logging.debug(\"Extracting database descriptors for evaluation/testing\")\n",
        "        database_subset_ds = Subset(eval_ds, list(range(eval_ds.database_num)))                       # subset del dataset da valutare non considerando le immagini di query\n",
        "        database_dataloader = DataLoader(dataset=database_subset_ds, num_workers=args.num_workers,\n",
        "                                         batch_size=args.infer_batch_size, pin_memory=(args.device == \"cuda\"))    # creazione del dataloader in grado di iterare sul dataset\n",
        "        all_descriptors = np.empty((len(eval_ds), args.fc_output_dim), dtype=\"float32\")   \n",
        "                                # ritorna un vettore non inizializzato con una riga per ogni sample da valutare\n",
        "        for images, indices in tqdm(database_dataloader, ncols=100):                                              # è un numero di colonne pari alla dimensione di descrittori\n",
        "            images.to(args.device)\n",
        "            descriptors = model(\"features_extractor\", [images, \"global\"])                                          # mette le immagini su device e ne calcola il risultato del MODELLO -> i descrittori\n",
        "            descriptors = descriptors.cpu().numpy()                                                               # porta i descrittori su cpu e li traforma da tensori ad array\n",
        "            all_descriptors[indices.numpy(), :] = descriptors                                                     # riempie l'array mettendo ad ogni indice il descrittore calcolato\n",
        "        \n",
        "        logging.debug(\"Extracting queries descriptors for evaluation/testing using batch size 1\")\n",
        "        queries_infer_batch_size = 1                                                                              # sembra che venga valutata un'immagine per volta\n",
        "        queries_subset_ds = Subset(eval_ds, list(range(eval_ds.database_num, eval_ds.database_num+eval_ds.queries_num)))    # in questo caso, crea un subset con sole query\n",
        "        queries_dataloader = DataLoader(dataset=queries_subset_ds, num_workers=args.num_workers, batch_size=queries_infer_batch_size, pin_memory=(args.device == \"cuda\"))            # crea il dataloader associato a questo secondo subset\n",
        "        for images, indices in tqdm(queries_dataloader, ncols=100):                            \n",
        "            images.to(args.device)\n",
        "            descriptors = model(\"features_extractor\", [images, \"global\"])                         # fa lo stesso lavoro precedente, calcolando per ogni immagine di query il descrittore\n",
        "            descriptors = descriptors.cpu().numpy()\n",
        "            all_descriptors[indices.numpy(), :] = descriptors                 # rimepiendo il vettore all_descriptors \n",
        "    \n",
        "    queries_descriptors = all_descriptors[eval_ds.database_num:]              # divide i descrittori delle queries\n",
        "    database_descriptors = all_descriptors[:eval_ds.database_num]             # dai descrittori del database di immagini da classificare\n",
        "    \n",
        "    # Use a kNN to find predictions     ----    faiss (Facebook AI Similarity Search) è una libreria di Facebook che permette di effetuare una ricerca tra somiglianze in maniera efficiente\n",
        "                                                             # faiss.IndexFlatL2 misura la l2 distance (o distanza euclidea) tra tutti i vettori dati e il quey vector \n",
        "    faiss_index = faiss.IndexFlatL2(args.fc_output_dim)      # qui sembra che lo stia inizializzando con la dimensione dei descrittori  \n",
        "    faiss_index.add(database_descriptors)                    # dopodiché ci aggiunge tutti i descrittori delle immagini di test \n",
        "    del database_descriptors, all_descriptors                # elimina roba non piiù utile\n",
        "    \n",
        "    logging.debug(\"Calculating recalls\")\n",
        "    _, predictions = faiss_index.search(queries_descriptors, max(RECALL_VALUES))   # effettua la ricerca con i descrittori delle query con i valori di recall specificati\n",
        "                                                                        # questa parte quindi è svolta unicamente da questa libreria, che calcola la distanza euclidea (quindi la vicinanza)\n",
        "                                                                        # per ogni k (preso da RECALL_VALUES) immagini con le immagini di query. Più k è alto è più ho possibilità di prendere la \n",
        "                                                                        # più vicina (lo si vede dopo)\n",
        "    #### For each query, check if the predictions are correct\n",
        "    positives_per_query = eval_ds.get_positives()               # per ogni query, restituisce l'immagine reale del dataset più vicina (credo, devo ancora guardare test_dataset)\n",
        "    recalls = np.zeros(len(RECALL_VALUES))                      # vettore di recalls inizializzato a zero\n",
        "    for query_index, preds in enumerate(predictions):           # per ogni predizione, prende indice e relativa predizione\n",
        "        for i, n in enumerate(RECALL_VALUES):                   # per ogni valore delle recall values (sono 5 valori)\n",
        "            if np.any(np.in1d(preds[:n], positives_per_query[query_index])):    # controlla che ogni valore nel primo 1Darray (quindi penso descrittore, non immagine) sia contenuto \n",
        "                                                                                # nel secondo. Quindi per ogni n controlla se le predizioni fino ad n (le n più vicine) contengono \n",
        "                                                                                # la relativa immagine di query (np.any -> almeno 1)\n",
        "                recalls[i:] += 1                                                # se si, aumenta la relativa recall\n",
        "                break                                                           # ed esce perché tanto l'ha già trovata. Quindi si favoriscono recall più basse\n",
        "    # Divide by queries_num and multiply by 100, so the recalls are in percentages\n",
        "    recalls = recalls / eval_ds.queries_num * 100                               # valori di recall espressi in percentuale (cioè quante query in percentuale sono cadute in quel valore di recall)   \n",
        "    recalls_str = \", \".join([f\"R@{val}: {rec:.1f}\" for val, rec in zip(RECALL_VALUES, recalls)])     # valori di recall in stringa\n",
        "    return recalls, recalls_str, predictions\n",
        "\n",
        "\n",
        "base_transform = transforms.Compose([\n",
        "            transforms.ToTensor(),\n",
        "            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),    # stessa mean e std del train\n",
        "])\n",
        "\n",
        "def open_image(path):\n",
        "    return Image.open(path).convert(\"RGB\")\n",
        "\n",
        "def use_rerank(model, predictions, test_dataset, num_reranked_predictions=5, test_batch_size=16):\n",
        "    \"\"\"Compute the test by warping the query-prediction pairs.\n",
        "    \n",
        "    Parameters\n",
        "    ----------\n",
        "    model : network.Network\n",
        "    predictions : np.array of int, containing the first 20 predictions for each query, with shape [queries_num, 20].\n",
        "    test_dataset : dataset_geoloc.GeolocDataset, which contains the test-time images (queries and gallery).\n",
        "    num_reranked_predictions : int, how many predictions to re-rank.\n",
        "    test_batch_size : int.\n",
        "    \n",
        "    Returns\n",
        "    -------\n",
        "    recalls : np.array of int, containing R@1, R@5, r@10, r@20.\n",
        "    recalls_pretty_str : str, pretty-printed recalls\n",
        "    \"\"\"\n",
        "    \n",
        "    model = model.eval()\n",
        "    reranked_predictions = predictions.copy()\n",
        "    with torch.no_grad():\n",
        "        for num_q in tqdm(range(test_dataset.queries_num), desc=\"Testing\", ncols=100):\n",
        "\n",
        "            dot_prods_wqp = np.zeros((num_reranked_predictions))\n",
        "            query_path = test_dataset.queries_paths[num_q]\n",
        "\n",
        "            for i1 in range(0, num_reranked_predictions, test_batch_size):\n",
        "\n",
        "                batch_indexes = list(range(num_reranked_predictions))[i1:i1+test_batch_size]\n",
        "                current_batch_size = len(batch_indexes)\n",
        "                pil_image = open_image(query_path)\n",
        "                query = base_transform(pil_image)\n",
        "                query_repeated_twice = torch.repeat_interleave(query.unsqueeze(0), current_batch_size, 0)\n",
        "                \n",
        "                preds = []\n",
        "                for i in batch_indexes:\n",
        "                    pred_path = test_dataset.database_paths[predictions[num_q, i]]\n",
        "                    pil_image = open_image(pred_path)\n",
        "                    query = base_transform(pil_image)\n",
        "                    preds.append(query)\n",
        "                preds = torch.stack(preds)\n",
        "                \n",
        "                warped_pair = compute_warping(model, query_repeated_twice.cuda(), preds.cuda())\n",
        "                q_features = model(\"features_extractor\", [warped_pair[0], \"local\"])\n",
        "                p_features = model(\"features_extractor\", [warped_pair[1], \"local\"])\n",
        "                # Sum along all axes except for B. wqp stands for warped query-prediction\n",
        "                dot_prod_wqp = (q_features * p_features).sum(list(range(1, len(p_features.shape)))).cpu().numpy()\n",
        "                \n",
        "                dot_prods_wqp[i1:i1+test_batch_size] = dot_prod_wqp\n",
        "            \n",
        "            reranking_indexes = dot_prods_wqp.argsort()[::-1]\n",
        "            reranked_predictions[num_q, :num_reranked_predictions] = predictions[num_q][reranking_indexes]\n",
        "    \n",
        "    ground_truths = test_dataset.get_positives()\n",
        "    recalls = np.zeros(len(RECALL_VALUES))  \n",
        "    for query_index, preds in enumerate(reranked_predictions): \n",
        "        for i, n in enumerate(RECALL_VALUES):\n",
        "            if np.any(np.in1d(preds[:n], ground_truths[query_index])): \n",
        "                recalls[i:] += 1   \n",
        "                break  \n",
        "    recalls = recalls / test_dataset.queries_num * 100\n",
        "    recalls_str = \", \".join([f\"R@{val}: {rec:.1f}\" for val, rec in zip(RECALL_VALUES, recalls)])\n",
        "    return recalls, recalls_str\n",
        "```\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "QJPW8rSPDsjf"
      },
      "source": [
        "Add this line to parser.py\n",
        "\n",
        "\n",
        "```\n",
        "# Multi scale parameters \n",
        "    parser.add_argument(\"--multi_scale\", action='store_true', help=\"Use multi scale\")\n",
        "    parser.add_argument(\"--select_resolutions\", type=float, default=[0.526, 0.588, 1, 1.7, 1.9], nargs=\"+\", help=\"Usage: --select_resolution 1 2 4 6\")\n",
        "    parser.add_argument(\"--multi_scale_method\", type=str, default=None, choices=[\"avg\", \"sum\", \"max\", \"min\"],\n",
        "                        help=\"Usage:--multi_scale_method=avg\")\n",
        "\n",
        "parser.add_argument('--resize', type=int, default=[480, 640], nargs=2, help=\"Resizing shape for images (HxW).\")\n",
        "```\n",
        "\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "vOWjSTHEKYHm"
      },
      "source": [
        "check descriptors and feature_ms"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "gtq-DYIW1uo2"
      },
      "source": [
        "test - #q: 105; #db: 12771 >: R@1: 59.0, R@5: 76.2, R@10: 82.9, R@20: 85.7"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "92ZVUMw1NuJl"
      },
      "source": [
        "Backbone: resnet 18\n",
        "\n",
        "model: baseline (cosplace)\n",
        "\n",
        "grl: si"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "6CyzuIu8jz7f"
      },
      "source": [
        "## ResNet18 baseline + GRL (avg)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Fih8Shvgyz0W"
      },
      "source": [
        "2023-05-02 16:38:56   < test - #q: 105; #db: 12771 >: R@1: 59.0, R@5: 77.1, R@10: 82.9, R@20: 86.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QnmxmfBoC2a9",
        "outputId": "13662d5d-4f39-4b67-c2ab-c2c3e5d1dbd1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-02 16:06:52   /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-02 16:06:52   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-02 16:06:52   The outputs are being saved in logs/default/2023-05-02_16-06-52\n",
            "2023-05-02 16:06:52   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-02 16:06:52   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-02 16:06:54   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [31:59<00:00,  2.40s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 51.75it/s]\n",
            "2023-05-02 16:38:56   < test - #q: 105; #db: 12771 >: R@1: 59.0, R@5: 77.1, R@10: 82.9, R@20: 86.7\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "wxv1lPAry1MO"
      },
      "source": [
        "2023-05-02 17:12:34   < test - #q: 315; #db: 12771 >: R@1: 74.3, R@5: 86.3, R@10: 90.8, R@20: 93.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GFBi7sfXK-Ez",
        "outputId": "ab9873fb-1290-4e31-a9e2-d56da7415335"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-02 16:40:53   /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3\n",
            "2023-05-02 16:40:53   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-02 16:40:53   The outputs are being saved in logs/default/2023-05-02_16-40-53\n",
            "2023-05-02 16:40:53   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-02 16:40:53   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-02 16:40:56   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [31:32<00:00,  2.37s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:05<00:00, 53.28it/s]\n",
            "2023-05-02 17:12:34   < test - #q: 315; #db: 12771 >: R@1: 74.3, R@5: 86.3, R@10: 90.8, R@20: 93.7\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "aPw9RyKUy2Td"
      },
      "source": [
        "2023-05-02 18:21:51   < test - #q: 1000; #db: 27191 >: R@1: 54.3, R@5: 68.1, R@10: 72.1, R@20: 74.9"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5aUZE6jFK-WD",
        "outputId": "0fde4beb-a3fe-4b58-d78f-d8dd6e4e6e14"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-02 17:15:04   /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3\n",
            "2023-05-02 17:15:04   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-02 17:15:04   The outputs are being saved in logs/default/2023-05-02_17-15-04\n",
            "2023-05-02 17:15:04   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-02 17:15:04   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-02 17:15:06   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:06:29<00:00,  2.35s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:15<00:00, 63.40it/s]\n",
            "2023-05-02 18:21:51   < test - #q: 1000; #db: 27191 >: R@1: 54.3, R@5: 68.1, R@10: 72.1, R@20: 74.9\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "rs93v3VUj3v3"
      },
      "source": [
        "## ResNet18, Geowarp (avg)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "_qzHlDupY1CJ"
      },
      "source": [
        "Backbone: resnet 18\n",
        "\n",
        "model: geowarp (cosplace)\n",
        "\n",
        "grl: no"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_UcZHl0AiYlP",
        "outputId": "cedf7cf2-ec4e-40ca-e4b7-52809fc44821"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-16 15:49:49   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3\n",
            "2023-05-16 15:49:49   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-16 15:49:49   The outputs are being saved in logs/default/2023-05-16_15-49-49\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:00<00:00, 262MB/s]\n",
            "2023-05-16 15:49:56   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-16 15:49:56   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-16 15:49:56   Start testing\n",
            "2023-05-16 15:49:56   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [37:48<00:00,  2.84s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.65it/s]\n",
            "2023-05-16 16:27:48   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:18<00:00,  5.66it/s]\n",
            "2023-05-16 16:28:06   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 54.3, R@5: 74.3, R@10: 81.0, R@20: 85.7\n",
            "2023-05-16 16:28:06     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 64.8, R@5: 74.3, R@10: 81.0, R@20: 85.7\n",
            "2023-05-16 16:28:06   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hot1uAxIiegO",
        "outputId": "4f528e0c-95b7-4409-d79d-e75a196e0ca5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-16 16:28:11   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3\n",
            "2023-05-16 16:28:11   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-16 16:28:11   The outputs are being saved in logs/default/2023-05-16_16-28-11\n",
            "2023-05-16 16:28:13   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-16 16:28:13   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-16 16:28:14   Start testing\n",
            "2023-05-16 16:28:14   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [32:05<00:00,  2.41s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:04<00:00, 67.15it/s]\n",
            "2023-05-16 17:00:24   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:56<00:00,  5.61it/s]\n",
            "2023-05-16 17:01:20   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 72.1, R@5: 85.1, R@10: 88.3, R@20: 92.7\n",
            "2023-05-16 17:01:20     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 75.2, R@5: 85.1, R@10: 88.3, R@20: 92.7\n",
            "2023-05-16 17:01:20   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BBbpbVorie6u",
        "outputId": "c78eacac-e341-4f6e-beef-e494f0f3537a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-16 22:26:45   /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3\n",
            "2023-05-16 22:26:45   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-16 22:26:45   The outputs are being saved in logs/default/2023-05-16_22-26-45\n",
            "2023-05-16 22:26:47   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-16 22:26:47   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-16 22:26:47   Start testing\n",
            "2023-05-16 22:26:47   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:04:32<00:00,  2.28s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:16<00:00, 59.12it/s]\n",
            "2023-05-16 23:31:37   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:40<00:00,  6.25it/s]\n",
            "2023-05-16 23:34:18   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 50.6, R@5: 66.4, R@10: 71.7, R@20: 75.2\n",
            "2023-05-16 23:34:18     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 60.9, R@5: 66.4, R@10: 71.7, R@20: 75.2\n",
            "2023-05-16 23:34:18   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "0NMPIiOO1w8l"
      },
      "source": []
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "VWzfY81Mj613"
      },
      "source": [
        "## EfficientNetV2s, Geowarp (avg)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-oYkOaWW4tkF",
        "outputId": "f05cad2c-2808-4365-b08e-b883faa0cef7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-16 14:33:22   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_brightness 0.2 --night_test True\n",
            "2023-05-16 14:33:22   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-16 14:33:22   The outputs are being saved in logs/default/2023-05-16_14-33-22\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-16 14:33:23   Using efficient net v2s\n",
            "2023-05-16 14:33:23   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 14:33:26   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-16 14:33:26   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-16 14:33:26   Start testing\n",
            "2023-05-16 14:33:26   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [42:39<00:00,  3.20s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:05<00:00, 17.81it/s]\n",
            "2023-05-16 15:16:11   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:45<00:00,  2.28it/s]\n",
            "2023-05-16 15:16:57   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 69.5, R@5: 86.7, R@10: 91.4, R@20: 94.3\n",
            "2023-05-16 15:16:57     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 79.0, R@5: 86.7, R@10: 91.4, R@20: 94.3\n",
            "2023-05-16 15:16:57   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_brightness 0.2 --night_test True #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WGk_x9d0Zxba",
        "outputId": "1f31dda4-1cef-49e3-808c-9a42ea954e5c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 10:35:24   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --grl_param=0.3\n",
            "2023-05-18 10:35:24   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-18 10:35:24   The outputs are being saved in logs/default/2023-05-18_10-35-24\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-18 10:35:25   Using efficient net v2s\n",
            "2023-05-18 10:35:25   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 10:35:27   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-18 10:35:27   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 10:35:28   Start testing\n",
            "2023-05-18 10:35:28   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [40:46<00:00,  3.06s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:16<00:00, 19.15it/s]\n",
            "2023-05-18 11:16:31   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [02:13<00:00,  2.36it/s]\n",
            "2023-05-18 11:18:45   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 81.9, R@5: 92.7, R@10: 96.2, R@20: 97.8\n",
            "2023-05-18 11:18:45     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 86.0, R@5: 92.7, R@10: 96.2, R@20: 97.8\n",
            "2023-05-18 11:18:45   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5-Yn11MtZ03y",
        "outputId": "44dbebcc-cc11-41e4-ffaf-b04823703739"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 11:18:49   /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --grl_param=0.3\n",
            "2023-05-18 11:18:49   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-18 11:18:49   The outputs are being saved in logs/default/2023-05-18_11-18-49\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-18 11:18:50   Using efficient net v2s\n",
            "2023-05-18 11:18:50   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 11:18:54   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-18 11:18:54   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 11:18:54   Start testing\n",
            "2023-05-18 11:18:54   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:26:36<00:00,  3.06s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:42<00:00, 23.58it/s]\n",
            "2023-05-18 12:46:14   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [06:44<00:00,  2.47it/s]\n",
            "2023-05-18 12:52:59   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 61.0, R@5: 72.8, R@10: 77.4, R@20: 80.8\n",
            "2023-05-18 12:52:59     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 67.3, R@5: 72.8, R@10: 77.4, R@20: 80.8\n",
            "2023-05-18 12:52:59   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "cyG4z9rLZoz1"
      },
      "source": [
        "## EfficientNet V2s, baseline + GRL (avg)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "9_JyHXGPXU06"
      },
      "source": [
        "backbone: efficient net v2s\n",
        "\n",
        "model: baseline\n",
        "\n",
        "grl: si"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "2P5eEuW0143e"
      },
      "source": [
        "R@1: 68.6, R@5: 86.7, R@10: 91.4, R@20: 93.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a2f6yNDJZrdz",
        "outputId": "810b5903-fb82-47ee-8a0c-6d62675ed53e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-03 14:29:23   /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-03 14:29:23   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-03 14:29:23   The outputs are being saved in logs/default/2023-05-03_14-29-23\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:00<00:00, 241MB/s]\n",
            "2023-05-03 14:29:25   Using efficient net v2s\n",
            "2023-05-03 14:29:25   Model uses weights IMAGENET1K_V1\n",
            "2023-05-03 14:29:26   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-03 14:29:26   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-03 14:29:32   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [41:19<00:00,  3.10s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 26.92it/s]\n",
            "2023-05-03 15:10:55   < test - #q: 105; #db: 12771 >: R@1: 68.6, R@5: 86.7, R@10: 91.4, R@20: 93.3\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "3BrF_WG118sm"
      },
      "source": [
        "R@1: 79.7, R@5: 92.1, R@10: 94.0, R@20: 96.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_Mx_E5oYg0ot",
        "outputId": "49427bed-e7e5-437e-8b33-775f4ae344eb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-03 15:11:01   /content/eval.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3\n",
            "2023-05-03 15:11:01   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-03 15:11:01   The outputs are being saved in logs/default/2023-05-03_15-11-01\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-03 15:11:01   Using efficient net v2s\n",
            "2023-05-03 15:11:01   Model uses weights IMAGENET1K_V1\n",
            "2023-05-03 15:11:02   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-03 15:11:02   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-03 15:11:04   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [41:18<00:00,  3.10s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:10<00:00, 29.39it/s]\n",
            "2023-05-03 15:52:33   < test - #q: 315; #db: 12771 >: R@1: 81.9, R@5: 92.7, R@10: 95.9, R@20: 96.5\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "8NEberN21-rm"
      },
      "source": [
        "R@1: 61.9, R@5: 73.2, R@10: 77.3, R@20: 81.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2yziyP4lnYLV",
        "outputId": "ef7ebc2d-3482-46d6-d7ed-ef205b52e48e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-03 15:52:38   /content/eval.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3\n",
            "2023-05-03 15:52:38   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-03 15:52:38   The outputs are being saved in logs/default/2023-05-03_15-52-38\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-03 15:52:39   Using efficient net v2s\n",
            "2023-05-03 15:52:39   Model uses weights IMAGENET1K_V1\n",
            "2023-05-03 15:52:39   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-03 15:52:39   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-03 15:52:41   Test with multi-scale, the multi-scale method is: avg\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:25:38<00:00,  3.02s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:35<00:00, 27.79it/s]\n",
            "2023-05-03 17:18:57   < test - #q: 1000; #db: 27191 >: R@1: 62.9, R@5: 74.3, R@10: 77.9, R@20: 81.8\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "aYoBzWQqbjcK"
      },
      "source": [
        "# Step 9.1: Multi scale - SUM"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "zE9-ApvP6Q3R"
      },
      "source": [
        "## Resnet18 baseline + GRL - (sum)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5tjmyAOr6VXq",
        "outputId": "02a2362e-adfc-441e-c883-c0967310ce19"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-17 15:40:43   /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-17 15:40:43   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-17 15:40:43   The outputs are being saved in logs/default/2023-05-17_15-40-43\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:03<00:00, 15.2MB/s]\n",
            "2023-05-17 15:40:46   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-17 15:40:46   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-17 15:40:51   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "  1%|▎                                                            | 4/799 [00:23<1:17:38,  5.86s/it]\n",
            "2023-05-17 15:41:15   \n",
            "Traceback (most recent call last):\n",
            "  File \"/content/eval.py\", line 50, in <module>\n",
            "    recalls, recalls_str = test.test(args, test_ds, model)\n",
            "  File \"/content/test.py\", line 65, in test\n",
            "    tmp_query = tra(original).to(args.device)  # transforming the img\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py\", line 217, in forward\n",
            "    input = module(input)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torchvision/transforms/transforms.py\", line 361, in forward\n",
            "    return F.resize(img, self.size, self.interpolation, self.max_size, self.antialias)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py\", line 492, in resize\n",
            "    return F_t.resize(img, size=output_size, interpolation=interpolation.value, antialias=antialias)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torchvision/transforms/_functional_tensor.py\", line 467, in resize\n",
            "    img = interpolate(img, size=size, mode=interpolation, align_corners=align_corners, antialias=antialias)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/functional.py\", line 3959, in interpolate\n",
            "    return torch._C._nn.upsample_bilinear2d(input, output_size, align_corners, scale_factors)\n",
            "KeyboardInterrupt\n",
            "\n",
            "2023-05-17 15:41:15   Experiment finished (with some errors)\n",
            "^C\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xpI-rXm880tL",
        "outputId": "942967ca-d0a8-4158-c755-b2afbdceeec8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-04 13:50:50   /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-04 13:50:50   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-04 13:50:50   The outputs are being saved in logs/default/2023-05-04_13-50-50\n",
            "2023-05-04 13:50:50   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-04 13:50:50   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-04 13:50:52   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [31:30<00:00,  2.37s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:05<00:00, 53.97it/s]\n",
            "2023-05-04 14:22:29   < test - #q: 315; #db: 12771 >: R@1: 71.1, R@5: 84.1, R@10: 88.3, R@20: 91.7\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N9Tx-u2W81EK",
        "outputId": "4a158fda-e559-4cdb-cb1d-1d71ceed556c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-04 14:22:32   /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-04 14:22:32   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-04 14:22:32   The outputs are being saved in logs/default/2023-05-04_14-22-32\n",
            "2023-05-04 14:22:33   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-04 14:22:33   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-04 14:22:35   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:05:43<00:00,  2.32s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:15<00:00, 64.23it/s]\n",
            "2023-05-04 15:28:34   < test - #q: 1000; #db: 27191 >: R@1: 50.5, R@5: 65.0, R@10: 69.8, R@20: 75.4\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "1enY_d91BC_k"
      },
      "source": [
        "## Resnet18, GeoWarp - (sum)\n",
        "\n",
        "> Blocco con rientro\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZE-jNSEpBJVK",
        "outputId": "8e29fb99-61d5-4090-f3ee-6c63b78cd520"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-17 15:50:10   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3\n",
            "2023-05-17 15:50:10   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-17 15:50:10   The outputs are being saved in logs/default/2023-05-17_15-50-10\n",
            "2023-05-17 15:50:13   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-17 15:50:13   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-17 15:50:14   Start testing\n",
            "2023-05-17 15:50:14   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [31:15<00:00,  2.35s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:04<00:00, 22.12it/s]\n",
            "2023-05-17 16:21:34   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:18<00:00,  5.74it/s]\n",
            "2023-05-17 16:21:52   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 44.8, R@5: 65.7, R@10: 76.2, R@20: 83.8\n",
            "2023-05-17 16:21:52     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 57.1, R@5: 65.7, R@10: 76.2, R@20: 83.8\n",
            "2023-05-17 16:21:52   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IAI7RMS8BJM6",
        "outputId": "06842a9c-b5fe-4899-83dc-ac75468b0cde"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 12:53:03   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3\n",
            "2023-05-18 12:53:03   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-18 12:53:03   The outputs are being saved in logs/default/2023-05-18_12-53-03\n",
            "2023-05-18 12:53:05   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-18 12:53:05   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-18 12:53:05   Start testing\n",
            "2023-05-18 12:53:05   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [33:05<00:00,  2.49s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:10<00:00, 29.31it/s]\n",
            "2023-05-18 13:26:22   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:57<00:00,  5.48it/s]\n",
            "2023-05-18 13:27:19   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 62.9, R@5: 77.8, R@10: 85.7, R@20: 90.2\n",
            "2023-05-18 13:27:19     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 72.1, R@5: 77.8, R@10: 85.7, R@20: 90.2\n",
            "2023-05-18 13:27:19   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ujFp2mWVBJD7",
        "outputId": "5cfdf84a-50d2-4dc5-ee3f-8f3aa77fc5f9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 13:47:27   /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3\n",
            "2023-05-18 13:47:27   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-18 13:47:27   The outputs are being saved in logs/default/2023-05-18_13-47-27\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:00<00:00, 246MB/s]\n",
            "2023-05-18 13:47:35   There are 1 GPUs and 4 CPUs.\n",
            "2023-05-18 13:47:35   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-18 13:47:35   Start testing\n",
            "2023-05-18 13:47:35   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [22:06<00:00,  1.28it/s]\n",
            "100%|██████████████████████████████████████████████████████████| 1000/1000 [00:08<00:00, 114.50it/s]\n",
            "2023-05-18 14:09:51   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [01:17<00:00, 12.96it/s]\n",
            "2023-05-18 14:11:08   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 47.1, R@5: 64.6, R@10: 70.4, R@20: 75.6\n",
            "2023-05-18 14:11:08     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 59.1, R@5: 64.6, R@10: 70.4, R@20: 75.6\n",
            "2023-05-18 14:11:08   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "odZ7rt1hBiPk"
      },
      "source": [
        "## EfficientNetV2s, Geowarp - (sum)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zr4Mdr4qBn30",
        "outputId": "bd05b0a6-9837-4463-985a-d626e80d5819"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 14:11:13   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_brightness 0.2 --night_test True\n",
            "2023-05-18 14:11:13   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=None, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-18 14:11:13   The outputs are being saved in logs/default/2023-05-18_14-11-13\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:00<00:00, 90.0MB/s]\n",
            "2023-05-18 14:11:15   Using efficient net v2s\n",
            "2023-05-18 14:11:15   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 14:11:17   There are 1 GPUs and 4 CPUs.\n",
            "2023-05-18 14:11:17   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 14:11:18   Start testing\n",
            "2023-05-18 14:11:18   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [13:22<00:00,  1.00s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 31.17it/s]\n",
            "2023-05-18 14:24:44   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:17<00:00,  6.06it/s]\n",
            "2023-05-18 14:25:01   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 67.6, R@5: 81.9, R@10: 87.6, R@20: 93.3\n",
            "2023-05-18 14:25:01     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 73.3, R@5: 81.9, R@10: 87.6, R@20: 93.3\n",
            "2023-05-18 14:25:01   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_brightness 0.2 --night_test True #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "99CsZXFOBnw6",
        "outputId": "9f6bded1-b57c-4ba5-82c3-e427ee732ea8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 14:40:07   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-05-18 14:40:07   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-18 14:40:07   The outputs are being saved in logs/default/2023-05-18_14-40-07\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:02<00:00, 42.5MB/s]\n",
            "2023-05-18 14:40:09   Using efficient net v2s\n",
            "2023-05-18 14:40:09   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 14:40:15   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 14:40:15   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 14:40:15   Start testing\n",
            "2023-05-18 14:40:15   Test with multi-scale, the multi-scale method is: sum\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:04<00:00,  1.88it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 37.55it/s]\n",
            "2023-05-18 14:47:28   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:39<00:00,  7.90it/s]\n",
            "2023-05-18 14:48:08   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 74.6, R@5: 89.8, R@10: 92.1, R@20: 95.2\n",
            "2023-05-18 14:48:08     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 81.3, R@5: 89.8, R@10: 92.1, R@20: 95.2\n",
            "2023-05-18 14:48:08   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TOIHs9qIBnoL",
        "outputId": "de7ebe28-91f7-4cb3-eb0d-2fd69a1a2b2b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 14:48:12   /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-05-18 14:48:12   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-18 14:48:12   The outputs are being saved in logs/default/2023-05-18_14-48-12\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-18 14:48:13   Using efficient net v2s\n",
            "2023-05-18 14:48:13   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 14:48:15   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 14:48:15   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 14:48:16   Start testing\n",
            "2023-05-18 14:48:16   Test with multi-scale, the multi-scale method is: sum\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:55<00:00,  1.90it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:27<00:00, 36.28it/s]\n",
            "2023-05-18 15:03:39   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:10<00:00,  7.68it/s]\n",
            "2023-05-18 15:05:49   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 60.6, R@5: 73.0, R@10: 77.7, R@20: 81.0\n",
            "2023-05-18 15:05:49     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 67.6, R@5: 73.0, R@10: 77.7, R@20: 81.0\n",
            "2023-05-18 15:05:49   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s  #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "3vQ5s3TGBvuq"
      },
      "source": [
        "## EfficientNetV2s, baseline + GRL (sum)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "61hSahLXB0iH",
        "outputId": "87001e94-4a48-424b-f911-75ebf4ba74e2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-04 19:08:39   /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-04 19:08:39   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-04 19:08:39   The outputs are being saved in logs/default/2023-05-04_19-08-39\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:01<00:00, 83.9MB/s]\n",
            "2023-05-04 19:08:41   Using efficient net v2s\n",
            "2023-05-04 19:08:41   Model uses weights IMAGENET1K_V1\n",
            "2023-05-04 19:08:42   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-04 19:08:42   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-04 19:08:49   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [39:51<00:00,  2.99s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 29.22it/s]\n",
            "2023-05-04 19:48:44   < test - #q: 105; #db: 12771 >: R@1: 67.6, R@5: 82.9, R@10: 89.5, R@20: 91.4\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bp356GlgB0db",
        "outputId": "ba411ce1-2b2b-45d8-fbc1-e637eebc8837"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-04 19:48:49   /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-04 19:48:49   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-04 19:48:49   The outputs are being saved in logs/default/2023-05-04_19-48-49\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-04 19:48:49   Using efficient net v2s\n",
            "2023-05-04 19:48:49   Model uses weights IMAGENET1K_V1\n",
            "2023-05-04 19:48:50   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-04 19:48:50   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-04 19:48:52   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [39:17<00:00,  2.95s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:11<00:00, 28.05it/s]\n",
            "2023-05-04 20:28:21   < test - #q: 315; #db: 12771 >: R@1: 78.4, R@5: 91.1, R@10: 94.3, R@20: 95.9\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tqG3E4TsB0Yb",
        "outputId": "a2337c32-a382-459a-c2cc-930e6a57c684"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-05 08:26:00   /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-05 08:26:00   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-05 08:26:00   The outputs are being saved in logs/default/2023-05-05_08-26-00\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:01<00:00, 83.6MB/s]\n",
            "2023-05-05 08:26:02   Using efficient net v2s\n",
            "2023-05-05 08:26:02   Model uses weights IMAGENET1K_V1\n",
            "2023-05-05 08:26:03   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-05 08:26:03   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-05 08:26:09   Test with multi-scale, the multi-scale method is: sum\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:24:00<00:00,  2.97s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:31<00:00, 31.34it/s]\n",
            "2023-05-05 09:50:43   < test - #q: 1000; #db: 27191 >: R@1: 60.6, R@5: 74.2, R@10: 77.2, R@20: 80.7\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "P7kz3lzkRXwH"
      },
      "source": [
        "# Step 9.2: Multi scale - MAX"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "_WPlVjQERbbe"
      },
      "source": [
        "## Resnet18 baseline + GRL (max)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RpOMDsTdRe2m",
        "outputId": "9532850f-b29b-4377-c155-7c974980a517"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-05 14:21:11   /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-05 14:21:11   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-05 14:21:11   The outputs are being saved in logs/default/2023-05-05_14-21-11\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:00<00:00, 114MB/s]\n",
            "2023-05-05 14:21:12   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-05 14:21:12   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-05 14:21:18   Test with multi-scale, the multi-scale method is: max\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [32:31<00:00,  2.44s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 50.04it/s]\n",
            "2023-05-05 14:53:52   < test - #q: 105; #db: 12771 >: R@1: 59.0, R@5: 75.2, R@10: 83.8, R@20: 85.7\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2phyIGNuRfae",
        "outputId": "5bc57307-1915-425d-ec23-98a6228a5377"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-05 14:53:58   /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-05 14:53:58   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-05 14:53:58   The outputs are being saved in logs/default/2023-05-05_14-53-58\n",
            "2023-05-05 14:53:58   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-05 14:53:58   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-05 14:54:00   Test with multi-scale, the multi-scale method is: max\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [31:56<00:00,  2.40s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:06<00:00, 50.44it/s]\n",
            "2023-05-05 15:26:03   < test - #q: 315; #db: 12771 >: R@1: 73.0, R@5: 85.4, R@10: 89.5, R@20: 92.1\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wNJLWKGvRfP-",
        "outputId": "ee469143-08b3-4c88-bae3-4e37094c9050"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-06 16:31:34   /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-06 16:31:34   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-06 16:31:34   The outputs are being saved in logs/default/2023-05-06_16-31-34\n",
            "2023-05-06 16:31:34   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-06 16:31:34   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-06 16:31:36   Test with multi-scale, the multi-scale method is: max\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:05:43<00:00,  2.32s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:15<00:00, 66.11it/s]\n",
            "2023-05-06 17:37:35   < test - #q: 1000; #db: 27191 >: R@1: 50.1, R@5: 64.0, R@10: 69.9, R@20: 74.2\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "q75MrhmNRvpv"
      },
      "source": [
        "## Resnet18, Geowarp (max)\n",
        "\n",
        "> Blocco con rientro\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L267e7-aRzne",
        "outputId": "e78fdbe5-5292-48e2-b9ce-7a3ad375b2e3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-17 16:21:57   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3\n",
            "2023-05-17 16:21:57   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-17 16:21:57   The outputs are being saved in logs/default/2023-05-17_16-21-57\n",
            "2023-05-17 16:22:00   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-17 16:22:00   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-17 16:22:00   Start testing\n",
            "2023-05-17 16:22:00   Test with multi-scale, the multi-scale method is: max\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [31:31<00:00,  2.37s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:05<00:00, 19.27it/s]\n",
            "2023-05-17 16:53:37   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:18<00:00,  5.80it/s]\n",
            "2023-05-17 16:53:55   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 53.3, R@5: 72.4, R@10: 83.8, R@20: 87.6\n",
            "2023-05-17 16:53:55     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 72.4, R@10: 83.8, R@20: 87.6\n",
            "2023-05-17 16:53:55   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ck0RPhsRzd_",
        "outputId": "b43d9785-3c64-4d62-8deb-eeb5e33f155b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 16:16:18   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3\n",
            "2023-05-18 16:16:18   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-18 16:16:18   The outputs are being saved in logs/default/2023-05-18_16-16-18\n",
            "2023-05-18 16:16:20   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 16:16:20   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-18 16:16:20   Start testing\n",
            "2023-05-18 16:16:20   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [05:26<00:00,  2.44it/s]\n",
            "100%|████████████████████████████████████████████████████████████| 315/315 [00:02<00:00, 138.15it/s]\n",
            "2023-05-18 16:21:50   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:17<00:00, 17.62it/s]\n",
            "2023-05-18 16:22:07   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 70.5, R@5: 84.8, R@10: 90.2, R@20: 92.7\n",
            "2023-05-18 16:22:07     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 77.8, R@5: 84.8, R@10: 90.2, R@20: 92.7\n",
            "2023-05-18 16:22:07   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V4awQMq2RzJ4",
        "outputId": "5113c45d-4991-4d4a-b68f-a24ed2e19c5f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 15:06:57   /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3\n",
            "2023-05-18 15:06:57   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-18 15:06:57   The outputs are being saved in logs/default/2023-05-18_15-06-57\n",
            "2023-05-18 15:06:59   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 15:06:59   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-18 15:07:00   Start testing\n",
            "2023-05-18 15:07:00   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [11:25<00:00,  2.48it/s]\n",
            "100%|██████████████████████████████████████████████████████████| 1000/1000 [00:07<00:00, 132.33it/s]\n",
            "2023-05-18 15:18:33   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [00:54<00:00, 18.18it/s]\n",
            "2023-05-18 15:19:28   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 51.0, R@5: 65.3, R@10: 69.8, R@20: 74.7\n",
            "2023-05-18 15:19:28     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 60.2, R@5: 65.3, R@10: 69.8, R@20: 74.7\n",
            "2023-05-18 15:19:28   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Y_8ig-u0R6xm"
      },
      "source": [
        "## EfficientNetV2s, Geowarp (max)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0_l2Uus0SDP2",
        "outputId": "3d412378-a7b6-43c3-f7f3-4ce37218f628"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 16:24:03   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_brightness 0.2 --night_test True\n",
            "2023-05-18 16:24:03   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=None, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-18 16:24:03   The outputs are being saved in logs/default/2023-05-18_16-24-03\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-18 16:24:04   Using efficient net v2s\n",
            "2023-05-18 16:24:04   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 16:24:06   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 16:24:06   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 16:24:07   Start testing\n",
            "2023-05-18 16:24:07   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:02<00:00,  1.89it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 33.66it/s]\n",
            "2023-05-18 16:31:12   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:13<00:00,  7.67it/s]\n",
            "2023-05-18 16:31:26   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 67.6, R@5: 81.9, R@10: 91.4, R@20: 95.2\n",
            "2023-05-18 16:31:26     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 76.2, R@5: 81.9, R@10: 91.4, R@20: 95.2\n",
            "2023-05-18 16:31:26   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_brightness 0.2 --night_test True #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i7naNogxSDG-",
        "outputId": "ebb2f170-695d-48c7-a848-b452be95da3a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 23:31:50   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-05-18 23:31:50   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-18 23:31:50   The outputs are being saved in logs/default/2023-05-18_23-31-50\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-18 23:31:50   Using efficient net v2s\n",
            "2023-05-18 23:31:50   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 23:31:52   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 23:31:52   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 23:31:53   Start testing\n",
            "2023-05-18 23:31:53   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:00<00:00,  1.90it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 39.07it/s]\n",
            "2023-05-18 23:39:01   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:40<00:00,  7.79it/s]\n",
            "2023-05-18 23:39:41   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 80.3, R@5: 92.4, R@10: 95.2, R@20: 97.5\n",
            "2023-05-18 23:39:41     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 85.4, R@5: 92.4, R@10: 95.2, R@20: 97.5\n",
            "2023-05-18 23:39:41   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s  #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mBy-DjRdSC4f",
        "outputId": "76d38a68-4a1f-4e29-a418-9d629172f438"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 15:20:26   /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-05-18 15:20:26   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-18 15:20:26   The outputs are being saved in logs/default/2023-05-18_15-20-26\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-18 15:20:27   Using efficient net v2s\n",
            "2023-05-18 15:20:27   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 15:20:29   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 15:20:29   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 15:20:30   Start testing\n",
            "2023-05-18 15:20:30   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:49<00:00,  1.91it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:27<00:00, 36.15it/s]\n",
            "2023-05-18 15:35:48   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:10<00:00,  7.67it/s]\n",
            "2023-05-18 15:37:58   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 58.9, R@5: 71.9, R@10: 75.8, R@20: 79.8\n",
            "2023-05-18 15:37:58     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 66.5, R@5: 71.9, R@10: 75.8, R@20: 79.8\n",
            "2023-05-18 15:37:58   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s  #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "AI-iJvneSLu3"
      },
      "source": [
        "## EfficientNetV2s, baseline + GRL (max)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tJqjnDL9SQ54",
        "outputId": "63c84e56-ef0f-440f-8747-00d15558136f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-07 14:04:36   /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-07 14:04:36   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-07 14:04:36   The outputs are being saved in logs/default/2023-05-07_14-04-36\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:01<00:00, 61.3MB/s]\n",
            "2023-05-07 14:04:39   Using efficient net v2s\n",
            "2023-05-07 14:04:39   Model uses weights IMAGENET1K_V1\n",
            "2023-05-07 14:04:39   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-07 14:04:39   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-07 14:04:45   Test with multi-scale, the multi-scale method is: max\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [39:10<00:00,  2.94s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 26.61it/s]\n",
            "2023-05-07 14:44:00   < test - #q: 105; #db: 12771 >: R@1: 69.5, R@5: 84.8, R@10: 91.4, R@20: 93.3\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HmAPfd72SQxf",
        "outputId": "b5f1dee8-f9a9-4cd2-8454-8c9219ca9861"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-07 14:44:04   /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-07 14:44:04   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-07 14:44:04   The outputs are being saved in logs/default/2023-05-07_14-44-04\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-07 14:44:05   Using efficient net v2s\n",
            "2023-05-07 14:44:05   Model uses weights IMAGENET1K_V1\n",
            "2023-05-07 14:44:05   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-07 14:44:05   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-07 14:44:08   Test with multi-scale, the multi-scale method is: max\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [39:34<00:00,  2.97s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:11<00:00, 28.23it/s]\n",
            "2023-05-07 15:23:53   < test - #q: 315; #db: 12771 >: R@1: 78.4, R@5: 92.1, R@10: 95.9, R@20: 96.8\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IgiPhbF6SQl_",
        "outputId": "2dee6ed8-24a4-4057-d470-75e20ced9c5c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-07 15:23:57   /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-07 15:23:57   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-07 15:23:57   The outputs are being saved in logs/default/2023-05-07_15-23-57\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-07 15:23:57   Using efficient net v2s\n",
            "2023-05-07 15:23:57   Model uses weights IMAGENET1K_V1\n",
            "2023-05-07 15:23:58   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-07 15:23:58   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-07 15:24:00   Test with multi-scale, the multi-scale method is: max\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:24:00<00:00,  2.97s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:36<00:00, 27.53it/s]\n",
            "2023-05-07 16:48:38   < test - #q: 1000; #db: 27191 >: R@1: 60.6, R@5: 72.1, R@10: 76.0, R@20: 80.4\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "H6kvtmzMTXOP"
      },
      "source": [
        "# Step 9.3: Multi scale - MIN"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "T6GL4ABETcVG"
      },
      "source": [
        "## Resnet18 baseline + GRL (min)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tyWgOm0KTrQK",
        "outputId": "3b6be19f-283d-4230-c1e6-be6ca17b9168"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-07 17:44:23   /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-07 17:44:23   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-07 17:44:23   The outputs are being saved in logs/default/2023-05-07_17-44-23\n",
            "2023-05-07 17:44:23   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-07 17:44:23   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-07 17:44:25   Test with multi-scale, the multi-scale method is: min\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [34:15<00:00,  2.57s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 27.88it/s]\n",
            "2023-05-07 18:18:45   < test - #q: 105; #db: 12771 >: R@1: 59.0, R@5: 76.2, R@10: 83.8, R@20: 86.7\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EYH-70IdTrJW",
        "outputId": "500ca796-f3a1-4758-fcd3-e9039197ee7b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-08 10:38:51   /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-08 10:38:51   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-08 10:38:51   The outputs are being saved in logs/default/2023-05-08_10-38-51\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:00<00:00, 291MB/s]\n",
            "2023-05-08 10:38:52   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-08 10:38:52   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-08 10:38:57   Test with multi-scale, the multi-scale method is: min\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [31:43<00:00,  2.38s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:04<00:00, 70.15it/s]\n",
            "2023-05-08 11:10:45   < test - #q: 315; #db: 12771 >: R@1: 71.4, R@5: 84.4, R@10: 89.5, R@20: 92.4\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_LwZi5zuTrBe",
        "outputId": "1c69eb18-b89e-4fa8-e199-584b3e4f2cd1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-08 12:33:41   /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-08 12:33:41   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-08 12:33:41   The outputs are being saved in logs/default/2023-05-08_12-33-41\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:00<00:00, 165MB/s]\n",
            "2023-05-08 12:33:41   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-08 12:33:41   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "2023-05-08 12:33:47   Test with multi-scale, the multi-scale method is: min\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:05:35<00:00,  2.32s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:15<00:00, 66.29it/s]\n",
            "2023-05-08 13:39:38   < test - #q: 1000; #db: 27191 >: R@1: 50.5, R@5: 63.7, R@10: 69.3, R@20: 74.0\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param=0.3 --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Bz-vkHNfThH-"
      },
      "source": [
        "## Resnet18, Geowarp (min)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OCG7Rz1fT4TH",
        "outputId": "ccea3bbc-167d-40b2-8d1b-8f913121097c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-17 16:53:59   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3\n",
            "2023-05-17 16:53:59   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-17 16:53:59   The outputs are being saved in logs/default/2023-05-17_16-53-59\n",
            "2023-05-17 16:54:02   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-17 16:54:02   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-17 16:54:02   Start testing\n",
            "2023-05-17 16:54:02   Test with multi-scale, the multi-scale method is: min\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [31:16<00:00,  2.35s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:05<00:00, 19.80it/s]\n",
            "2023-05-17 17:25:24   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:18<00:00,  5.57it/s]\n",
            "2023-05-17 17:25:43   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 51.4, R@5: 70.5, R@10: 79.0, R@20: 84.8\n",
            "2023-05-17 17:25:43     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 70.5, R@10: 79.0, R@20: 84.8\n",
            "2023-05-17 17:25:43   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --night_brightness 0.2 --night_test True --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e9Wl_zwxT4TI",
        "outputId": "e57b1a5f-a624-4ee9-d1de-d01a3cea0c77"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 23:42:11   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3\n",
            "2023-05-18 23:42:11   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-18 23:42:11   The outputs are being saved in logs/default/2023-05-18_23-42-11\n",
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100% 44.7M/44.7M [00:00<00:00, 378MB/s]\n",
            "2023-05-18 23:42:13   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 23:42:13   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-18 23:42:13   Start testing\n",
            "2023-05-18 23:42:13   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [05:22<00:00,  2.48it/s]\n",
            "100%|████████████████████████████████████████████████████████████| 315/315 [00:02<00:00, 140.30it/s]\n",
            "2023-05-18 23:47:38   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:17<00:00, 18.03it/s]\n",
            "2023-05-18 23:47:56   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 68.9, R@5: 82.9, R@10: 87.9, R@20: 90.5\n",
            "2023-05-18 23:47:56     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 74.0, R@5: 82.9, R@10: 87.9, R@20: 90.5\n",
            "2023-05-18 23:47:56   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth  --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pfLcJCh0T4TJ",
        "outputId": "fe8bccd8-05e2-4be3-9dd1-38644ecca6ea"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 00:17:15   /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth --grl_param=0.3\n",
            "2023-05-19 00:17:15   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/geowarp_model/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-19 00:17:15   The outputs are being saved in logs/default/2023-05-19_00-17-15\n",
            "2023-05-19 00:17:16   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 00:17:16   Loading model from /content/geowarp_model/best_model.pth\n",
            "2023-05-19 00:17:17   Start testing\n",
            "2023-05-19 00:17:17   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [11:28<00:00,  2.47it/s]\n",
            "100%|██████████████████████████████████████████████████████████| 1000/1000 [00:07<00:00, 136.75it/s]\n",
            "2023-05-19 00:28:54   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [00:54<00:00, 18.29it/s]\n",
            "2023-05-19 00:29:48   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 49.8, R@5: 64.5, R@10: 70.4, R@20: 74.4\n",
            "2023-05-19 00:29:48     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 59.4, R@5: 64.5, R@10: 70.4, R@20: 74.4\n",
            "2023-05-19 00:29:48   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/geowarp_model/best_model.pth  --grl_param=0.3 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "uRHK3qCTTi-W"
      },
      "source": [
        "## EfficientNetV2s, Geowarp (min)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z-ffQSPkUPhb",
        "outputId": "6b1ab169-0b84-457b-dc7c-d2cfbdd7cde1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 00:39:22   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_brightness 0.2 --night_test True\n",
            "2023-05-19 00:39:22   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=None, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-19 00:39:22   The outputs are being saved in logs/default/2023-05-19_00-39-22\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 00:39:22   Using efficient net v2s\n",
            "2023-05-19 00:39:22   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 00:39:25   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 00:39:25   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 00:39:25   Start testing\n",
            "2023-05-19 00:39:25   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:02<00:00,  1.89it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 32.76it/s]\n",
            "2023-05-19 00:46:31   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:13<00:00,  7.66it/s]\n",
            "2023-05-19 00:46:44   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 66.7, R@5: 82.9, R@10: 91.4, R@20: 95.2\n",
            "2023-05-19 00:46:44     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 75.2, R@5: 82.9, R@10: 91.4, R@20: 95.2\n",
            "2023-05-19 00:46:44   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_brightness 0.2 --night_test True #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2H_8mgsqUPhc",
        "outputId": "a2499b45-ff26-476e-ffb0-0fa67aed0005"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 15:49:14   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-05-18 15:49:14   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-18 15:49:14   The outputs are being saved in logs/default/2023-05-18_15-49-14\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-18 15:49:14   Using efficient net v2s\n",
            "2023-05-18 15:49:14   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 15:49:17   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 15:49:17   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 15:49:17   Start testing\n",
            "2023-05-18 15:49:17   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:59<00:00,  1.90it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 36.46it/s]\n",
            "2023-05-18 15:56:25   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:40<00:00,  7.74it/s]\n",
            "2023-05-18 15:57:06   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 78.4, R@5: 89.5, R@10: 95.2, R@20: 97.5\n",
            "2023-05-18 15:57:06     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 83.2, R@5: 89.5, R@10: 95.2, R@20: 97.5\n",
            "2023-05-18 15:57:06   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s  #file in logs.zip "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xkd7i-0YUPhc",
        "outputId": "9ac689fe-0931-4b30-af0c-668763162860"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-18 15:57:10   /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-05-18 15:57:10   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-18 15:57:10   The outputs are being saved in logs/default/2023-05-18_15-57-10\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-18 15:57:10   Using efficient net v2s\n",
            "2023-05-18 15:57:10   Model uses weights IMAGENET1K_V1\n",
            "2023-05-18 15:57:13   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-18 15:57:13   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-18 15:57:13   Start testing\n",
            "2023-05-18 15:57:13   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:49<00:00,  1.91it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:28<00:00, 35.26it/s]\n",
            "2023-05-18 16:12:32   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:09<00:00,  7.72it/s]\n",
            "2023-05-18 16:14:42   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 59.5, R@5: 72.2, R@10: 75.9, R@20: 79.6\n",
            "2023-05-18 16:14:42     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 66.9, R@5: 72.2, R@10: 75.9, R@20: 79.6\n",
            "2023-05-18 16:14:42   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s  #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "rrFQn1C8Tn9u"
      },
      "source": [
        "## EfficientNetV2s, baseline + GRL (min)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "19wO3aQbUINV",
        "outputId": "46eefa8d-ec30-42ae-81df-0d7ffb94c75c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-08 13:39:58   /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-08 13:39:58   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-08 13:39:58   The outputs are being saved in logs/default/2023-05-08_13-39-58\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:01<00:00, 52.2MB/s]\n",
            "2023-05-08 13:40:01   Using efficient net v2s\n",
            "2023-05-08 13:40:01   Model uses weights IMAGENET1K_V1\n",
            "2023-05-08 13:40:02   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-08 13:40:02   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-08 13:40:04   Test with multi-scale, the multi-scale method is: min\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [38:51<00:00,  2.92s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:05<00:00, 19.54it/s]\n",
            "2023-05-08 14:19:01   < test - #q: 105; #db: 12771 >: R@1: 66.7, R@5: 85.7, R@10: 90.5, R@20: 93.3\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo-night --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_XuFwV1rUINW",
        "outputId": "ee51a2df-af63-4666-8680-0ba682aca3d7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-08 14:19:05   /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-08 14:19:05   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-08 14:19:05   The outputs are being saved in logs/default/2023-05-08_14-19-05\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-08 14:19:05   Using efficient net v2s\n",
            "2023-05-08 14:19:05   Model uses weights IMAGENET1K_V1\n",
            "2023-05-08 14:19:06   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-08 14:19:06   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-08 14:19:08   Test with multi-scale, the multi-scale method is: min\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [38:55<00:00,  2.92s/it]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:10<00:00, 30.32it/s]\n",
            "2023-05-08 14:58:14   < test - #q: 315; #db: 12771 >: R@1: 79.0, R@5: 91.7, R@10: 95.6, R@20: 96.8\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/tokyo_xs --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aftzf090UINW",
        "outputId": "3e037e1e-4cab-4ad5-f263-f601a19424d8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-08 14:58:17   /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-05-08 14:58:17   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-05-08 14:58:17   The outputs are being saved in logs/default/2023-05-08_14-58-17\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-08 14:58:17   Using efficient net v2s\n",
            "2023-05-08 14:58:17   Model uses weights IMAGENET1K_V1\n",
            "2023-05-08 14:58:18   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-08 14:58:18   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-08 14:58:21   Test with multi-scale, the multi-scale method is: min\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████| 1700/1700 [1:21:30<00:00,  2.88s/it]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:31<00:00, 31.33it/s]\n",
            "2023-05-08 16:20:24   < test - #q: 1000; #db: 27191 >: R@1: 60.3, R@5: 72.0, R@10: 75.5, R@20: 79.6\n"
          ]
        }
      ],
      "source": [
        "!python /content/eval.py --dataset_folder /content/small --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "bUgmgnYx4bYz"
      },
      "source": [
        "#Step 4: First Type of DA"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Td47wFfO2hh9"
      },
      "source": [
        "https://pastebin.com/2xT5G5Ga"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nFmBxaSD4hCA",
        "outputId": "fbc45a3e-d458-4a1f-ad47-3945473738c3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "______________________________________________________________FACTOR0.1\n",
            "_________________________STANDARD\n",
            "______________________________________________________________FACTOR0.1\n",
            "_________________________GRL\n",
            "2023-04-21 14:07:10   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True --night_brightness 0.1\n",
            "2023-04-21 14:07:10   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 14:07:10   The outputs are being saved in logs/default/2023-04-21_14-07-10\n",
            "2023-04-21 14:07:10   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 14:07:10   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:31<00:00,  5.26it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 34.79it/s]\n",
            "2023-04-21 14:09:47   < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 78.1, R@10: 81.9, R@20: 87.6\n",
            "______________________________________________________________FACTOR0.15\n",
            "_________________________STANDARD\n",
            "______________________________________________________________FACTOR0.15\n",
            "_________________________GRL\n",
            "2023-04-21 14:09:51   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True --night_brightness 0.15\n",
            "2023-04-21 14:09:51   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.15, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 14:09:51   The outputs are being saved in logs/default/2023-04-21_14-09-51\n",
            "2023-04-21 14:09:51   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 14:09:51   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:34<00:00,  5.17it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 45.90it/s]\n",
            "2023-04-21 14:12:30   < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 80.0, R@10: 83.8, R@20: 86.7\n",
            "______________________________________________________________FACTOR0.2\n",
            "_________________________STANDARD\n",
            "______________________________________________________________FACTOR0.2\n",
            "_________________________GRL\n",
            "2023-04-21 14:12:34   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-04-21 14:12:34   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 14:12:34   The outputs are being saved in logs/default/2023-04-21_14-12-34\n",
            "2023-04-21 14:12:34   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 14:12:34   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:31<00:00,  5.28it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 51.71it/s]\n",
            "2023-04-21 14:15:10   < test - #q: 105; #db: 12771 >: R@1: 58.1, R@5: 81.0, R@10: 83.8, R@20: 87.6\n",
            "______________________________________________________________FACTOR0.3\n",
            "_________________________STANDARD\n",
            "______________________________________________________________FACTOR0.3\n",
            "_________________________GRL\n",
            "2023-04-21 14:15:13   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True --night_brightness 0.3\n",
            "2023-04-21 14:15:13   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.3, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 14:15:13   The outputs are being saved in logs/default/2023-04-21_14-15-13\n",
            "2023-04-21 14:15:14   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 14:15:14   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:34<00:00,  5.16it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 52.32it/s]\n",
            "2023-04-21 14:17:53   < test - #q: 105; #db: 12771 >: R@1: 57.1, R@5: 80.0, R@10: 84.8, R@20: 88.6\n",
            "______________________________________________________________FACTOR0.4\n",
            "_________________________STANDARD\n",
            "______________________________________________________________FACTOR0.4\n",
            "_________________________GRL\n",
            "2023-04-21 14:17:56   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True --night_brightness 0.4\n",
            "2023-04-21 14:17:56   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/cosplace_with_grl/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.4, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 14:17:56   The outputs are being saved in logs/default/2023-04-21_14-17-56\n",
            "2023-04-21 14:17:56   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 14:17:56   Loading model from /content/logs/content/logs/default/cosplace_with_grl/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:31<00:00,  5.29it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 33.18it/s]\n",
            "2023-04-21 14:20:33   < test - #q: 105; #db: 12771 >: R@1: 60.0, R@5: 80.0, R@10: 84.8, R@20: 88.6\n"
          ]
        }
      ],
      "source": [
        "for factor in [0.1, 0.15, 0.2, 0.3, 0.4]:\n",
        "  print(f\"______________________________________________________________FACTOR{factor}\")\n",
        "  print(f\"_________________________STANDARD\")\n",
        "  #!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth --night_test True --night_brightness $factor #file in logs.zip\n",
        "  print(f\"______________________________________________________________FACTOR{factor}\")\n",
        "  print(f\"_________________________GRL\")\n",
        "  !python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/logs/content/logs/default/cosplace_with_grl/best_model.pth --grl_param 0.3 --night_test True --night_brightness $factor #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "CPYJfT3spvqV"
      },
      "source": [
        "# Evaluation with EffientNetV2s"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "y-5CP5dPp3h0"
      },
      "source": [
        "## EffientNetV2s + GRL on sf-xs(test)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "PP4gDmT82jx2"
      },
      "source": [
        "R@1: 63.0, R@5: 74.2, R@10: 78.7, R@20: 82.8\n",
        "\n",
        "21-04:  R@1: 63.4, R@5: 76.4, R@10: 79.0, R@20: 83.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p6AgSFlkhBQn",
        "outputId": "6d70beff-4364-430e-f042-ac286779cd0c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 11:12:56   eval.py --dataset_folder /content/small/ --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3\n",
            "2023-04-21 11:12:56   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-21 11:12:56   The outputs are being saved in logs/default/2023-04-21_11-12-56\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 11:12:57   Using efficient net v2s\n",
            "2023-04-21 11:12:57   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 11:12:58   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 11:12:58   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:33<00:00,  4.32it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:32<00:00, 31.17it/s]\n",
            "2023-04-21 11:20:06   < test - #q: 1000; #db: 27191 >: R@1: 63.4, R@5: 76.4, R@10: 79.0, R@20: 83.2\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/small/  --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "uyM4-CLHqAbW"
      },
      "source": [
        "## EffientNetV2s + GRL on tokyo_xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "tQc_OhGq2lYV"
      },
      "source": [
        "R@1: 63.0, R@5: 74.2, R@10: 78.7, R@20: 82.8\n",
        "\n",
        "21-04: R@1: 83.5, R@5: 93.0, R@10: 95.6, R@20: 97.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "odwFxSoL-0Zu",
        "outputId": "d7d8d132-427d-46a0-af78-38f87fcfd151"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 11:00:58   eval.py --dataset_folder /content/tokyo_xs/ --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3\n",
            "2023-04-21 11:00:58   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 11:00:58   The outputs are being saved in logs/default/2023-04-21_11-00-58\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:00<00:00, 129MB/s]\n",
            "2023-04-21 11:00:59   Using efficient net v2s\n",
            "2023-04-21 11:00:59   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 11:00:59   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 11:00:59   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:42<00:00,  3.59it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:09<00:00, 34.47it/s]\n",
            "2023-04-21 11:04:54   < test - #q: 315; #db: 12771 >: R@1: 83.5, R@5: 93.0, R@10: 95.6, R@20: 97.1\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo_xs/  --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "4lemTRHaqHyN"
      },
      "source": [
        "## EffientNetV2s + GRL on tokyo_night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "q9EhZw-_2nem"
      },
      "source": [
        "R@1: 73.3, R@5: 84.8, R@10: 90.5, R@20: 93.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p_FFesGy-497",
        "outputId": "4e948c84-aba9-469e-b2e8-d034019cb0be"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 11:04:58   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2\n",
            "2023-04-21 11:04:58   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_grl/eff2vs_grl.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 11:04:58   The outputs are being saved in logs/default/2023-04-21_11-04-58\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 11:04:59   Using efficient net v2s\n",
            "2023-04-21 11:04:59   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 11:04:59   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 11:04:59   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:47<00:00,  3.51it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:05<00:00, 20.66it/s]\n",
            "2023-04-21 11:08:54   < test - #q: 105; #db: 12771 >: R@1: 73.3, R@5: 84.8, R@10: 90.5, R@20: 93.3\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/eff2vs_grl/eff2vs_grl.pth --backbone efficientnet_v2_s --grl_param 0.3 --night_test True --night_brightness 0.2 #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "8MwD472tqNDW"
      },
      "source": [
        "# Step 4: Different backbone - Training of EfficientNetV2s (with no other change)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BIkMdKgtnfjH",
        "outputId": "035c40ad-60ea-454b-906d-89a99ecdbf89"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-02-27 16:09:55   /content/train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --backbone efficientnet_v2_s\n",
            "2023-02-27 16:09:55   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='efficientnet_v2_s', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=3, fc_output_dim=512, features_wise_w=10, grl_param=None, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10000, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method='avg', night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=8, optim='adam', positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resize=[480, 640], resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], source_dir=None, ss_w=1, target_dir=None, test_method='hard_resize', test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val', wd=None)\n",
            "2023-02-27 16:09:55   The outputs are being saved in logs/default/2023-02-27_16-09-55\n",
            "2023-02-27 16:09:55   Gradient Reversal Layer is disabled\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-02-27 16:09:55   Using efficient net v2s\n",
            "2023-02-27 16:09:55   Model uses weights IMAGENET1K_V1\n",
            "2023-02-27 16:09:56   Train last two layers of EfficientNet, freeze the previous ones\n",
            "2023-02-27 16:09:57   There are 1 GPUs and 12 CPUs.\n",
            "2023-02-27 16:09:58   Using cached dataset cache/small_M10_N5_mipc10.torch\n",
            "2023-02-27 16:09:58   Using cosface loss function.\n",
            "2023-02-27 16:09:58   Using 1 groups\n",
            "2023-02-27 16:09:58   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-02-27 16:09:58   The 1 groups have respectively the following number of images [59650]\n",
            "2023-02-27 16:09:59   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-02-27 16:09:59   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-27 16:09:59   Start training ...\n",
            "2023-02-27 16:09:59   There are 5965 classes for the first group, each epoch has 10000 iterations with batch_size 32, therefore the model sees each class (on average) 53.6 times per epoch\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [33:43<00:00,  4.94it/s]\n",
            "2023-02-27 16:43:42   Epoch 00 in 0:33:43, loss = 8.8771\n",
            "2023-02-27 16:43:42   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:23<00:00, 21.11it/s]\n",
            "2023-02-27 16:44:06   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [05:44<00:00, 23.19it/s]\n",
            "2023-02-27 16:49:51   Calculating recalls\n",
            "2023-02-27 16:49:52   Epoch 00 in 0:39:53, < val - #q: 7993; #db: 8015 >: R@1: 83.1, R@5: 91.1\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [33:45<00:00,  4.94it/s]\n",
            "2023-02-27 17:23:39   Epoch 01 in 0:33:46, loss = 3.6247\n",
            "2023-02-27 17:23:39   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:23<00:00, 21.13it/s]\n",
            "2023-02-27 17:24:02   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [05:45<00:00, 23.15it/s]\n",
            "2023-02-27 17:29:48   Calculating recalls\n",
            "2023-02-27 17:29:49   Epoch 01 in 0:39:56, < val - #q: 7993; #db: 8015 >: R@1: 86.7, R@5: 93.2\n",
            "100%|█████████████████████████████████████████████████████████| 10000/10000 [33:47<00:00,  4.93it/s]\n",
            "2023-02-27 18:03:38   Epoch 02 in 0:33:48, loss = 2.4675\n",
            "2023-02-27 18:03:38   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:23<00:00, 21.34it/s]\n",
            "2023-02-27 18:04:02   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [05:46<00:00, 23.09it/s]\n",
            "2023-02-27 18:09:48   Calculating recalls\n",
            "2023-02-27 18:09:49   Epoch 02 in 0:39:59, < val - #q: 7993; #db: 8015 >: R@1: 87.8, R@5: 93.7\n",
            "2023-02-27 18:09:50   Trained for 03 epochs, in total in 1:59:54\n",
            "2023-02-27 18:09:50   Now testing on the test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-27 18:09:50   Extracting database descriptors for evaluation/testing\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [01:18<00:00, 21.70it/s]\n",
            "2023-02-27 18:11:08   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:42<00:00, 23.27it/s]\n",
            "2023-02-27 18:11:51   Calculating recalls\n",
            "2023-02-27 18:11:52   < test - #q: 1000; #db: 27191 >: R@1: 63.9, R@5: 74.7, R@10: 79.4, R@20: 82.9\n",
            "2023-02-27 18:11:52   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/train.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --backbone efficientnet_v2_s "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "fRm4SQi9hTmL"
      },
      "source": [
        "## Test with EfficientNetV2s (no GRL) "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "5xpCyStChaEK"
      },
      "source": [
        "### Test on sf-xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "lk63mmas2sQt"
      },
      "source": [
        "R@1: 62.5, R@5: 74.5, R@10: 78.3, R@20: 81.5\n",
        "\n",
        "21-04: R@1: 63.9, R@5: 74.8, R@10: 79.4, R@20: 82.9"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VD_j9DVQhb7s",
        "outputId": "dc9a7fda-5a72-4c92-8c6f-6a3fb104712b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 11:20:47   eval.py --dataset_folder /content/small/ --resume_model /content/eff2vs/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-04-21 11:20:47   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-21 11:20:47   The outputs are being saved in logs/default/2023-04-21_11-20-47\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 11:20:48   Using efficient net v2s\n",
            "2023-04-21 11:20:48   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 11:20:48   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 11:20:48   Loading model from /content/eff2vs/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:32<00:00,  4.33it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:32<00:00, 30.70it/s]\n",
            "2023-04-21 11:27:56   < test - #q: 1000; #db: 27191 >: R@1: 63.9, R@5: 74.8, R@10: 79.4, R@20: 82.9\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/small/  --resume_model /content/eff2vs/best_model.pth  --backbone efficientnet_v2_s #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "0xoGmmnfhcfC"
      },
      "source": [
        "### Test on tokyo-xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "2iyQ3AYYs2dF"
      },
      "source": [
        "Rinominare queries in queries_v1"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "HrTQPZED2uRF"
      },
      "source": [
        "R@1: 80.0, R@5: 90.8, R@10: 94.3, R@20: 96.2\n",
        "\n",
        "21-04: R@1: 83.8, R@5: 92.4, R@10: 95.6, R@20: 97.5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-bVwp_VwheHb",
        "outputId": "09133105-1f17-43e9-8572-3a925f503b08"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 11:28:00   eval.py --dataset_folder /content/tokyo_xs/ --resume_model /content/eff2vs/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-04-21 11:28:00   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 11:28:00   The outputs are being saved in logs/default/2023-04-21_11-28-00\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 11:28:00   Using efficient net v2s\n",
            "2023-04-21 11:28:00   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 11:28:01   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 11:28:01   Loading model from /content/eff2vs/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:47<00:00,  3.52it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:11<00:00, 28.33it/s]\n",
            "2023-04-21 11:32:01   < test - #q: 315; #db: 12771 >: R@1: 83.8, R@5: 92.4, R@10: 95.6, R@20: 97.5\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo_xs/  --resume_model /content/eff2vs/best_model.pth --backbone efficientnet_v2_s #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "uJv-xdKbheVy"
      },
      "source": [
        "### Test on tokyo-night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "qCFveac92wGt"
      },
      "source": [
        "R@1: 73.3, R@5: 87.6, R@10: 90.5, R@20: 94.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8F5x0Aj4hggC",
        "outputId": "58b9ed26-a8f2-456e-efdf-ad92264b63b2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 11:32:05   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/eff2vs/best_model.pth --backbone efficientnet_v2_s --night_test True --night_brightness 0.2\n",
            "2023-04-21 11:32:05   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 11:32:05   The outputs are being saved in logs/default/2023-04-21_11-32-05\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 11:32:05   Using efficient net v2s\n",
            "2023-04-21 11:32:05   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 11:32:06   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 11:32:06   Loading model from /content/eff2vs/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:47<00:00,  3.51it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 27.33it/s]\n",
            "2023-04-21 11:36:00   < test - #q: 105; #db: 12771 >: R@1: 73.3, R@5: 87.6, R@10: 90.5, R@20: 94.3\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/eff2vs/best_model.pth --backbone efficientnet_v2_s --night_test True --night_brightness 0.2 #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "x_KZGKnz2ybn"
      },
      "source": [
        "R@1: 70.5, R@5: 85.7, R@10: 91.4, R@20: 93.3\n",
        "\n",
        "21-04: R@1: 74.3, R@5: 88.6, R@10: 92.4, R@20: 95.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-vLXk1Seqix9",
        "outputId": "1501c3fe-9a43-4e3d-a910-c2235811e97a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 11:36:03   eval.py --dataset_folder /content/tokyo-night/ --resume_model /content/eff2vs/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-04-21 11:36:03   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 11:36:03   The outputs are being saved in logs/default/2023-04-21_11-36-03\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 11:36:04   Using efficient net v2s\n",
            "2023-04-21 11:36:04   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 11:36:04   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 11:36:04   Loading model from /content/eff2vs/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:47<00:00,  3.51it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:05<00:00, 20.15it/s]\n",
            "2023-04-21 11:39:59   < test - #q: 105; #db: 12771 >: R@1: 74.3, R@5: 88.6, R@10: 92.4, R@20: 95.2\n"
          ]
        }
      ],
      "source": [
        "!python eval.py --dataset_folder /content/tokyo-night/  --resume_model /content/eff2vs/best_model.pth --backbone efficientnet_v2_s "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "fUxjeTtBwY4u"
      },
      "source": [
        "# Step 4: Different backbone - EfficientNetV2s + GeoWarp"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "QaTVgI4zxYrv"
      },
      "source": [
        "## Training GeoWarp with EfficientNetV2s "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pOuSq3iExcqX",
        "outputId": "ed623c9c-bcca-41c4-9461-4a203294c6b8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-02-28 15:41:22   /content/train_geowarp.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --backbone efficientnet_v2_s\n",
            "2023-02-28 15:41:22   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='efficientnet_v2_s', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=3, fc_output_dim=512, features_wise_w=10, grl_param=None, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10000, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method='avg', night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=8, optim='adam', positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resize=[480, 640], resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], source_dir=None, ss_w=1, target_dir=None, test_method='hard_resize', test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val', wd=None)\n",
            "2023-02-28 15:41:22   The outputs are being saved in logs/default/2023-02-28_15-41-22\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:00<00:00, 208MB/s]\n",
            "2023-02-28 15:41:23   Using efficient net v2s\n",
            "2023-02-28 15:41:23   Model uses weights IMAGENET1K_V1\n",
            "2023-02-28 15:41:23   Train last two layers of EfficientNet, freeze the previous ones\n",
            "2023-02-28 15:41:24   There are 1 GPUs and 12 CPUs.\n",
            "2023-02-28 15:41:24   WARNING: --resume_fe is set to None, meaning that the Feature Extractor is not initialized!\n",
            "2023-02-28 15:41:28   Cached dataset cache/small_M10_N5_mipc10.torch does not exist, I'll create it now.\n",
            "2023-02-28 15:41:28   Searching training images in /content/small/train\n",
            "2023-02-28 15:41:28   Found 59650 images\n",
            "2023-02-28 15:41:28   For each image, get its UTM east, UTM north and heading from its path\n",
            "2023-02-28 15:41:28   For each image, get class and group to which it belongs\n",
            "2023-02-28 15:41:29   Group together images belonging to the same class\n",
            "2023-02-28 15:41:29   Group together classes belonging to the same group\n",
            "2023-02-28 15:41:29   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-02-28 15:41:29   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-28 15:41:29   Using cosface loss function.\n",
            "100%|███████████████████████████████████████████████████████| 10000/10000 [1:09:27<00:00,  2.40it/s]\n",
            "2023-02-28 16:50:57   Epoch 00 in 1:09:28, loss = 4.4542\n",
            "2023-02-28 16:50:57   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:26<00:00, 19.18it/s]\n",
            "2023-02-28 16:51:23   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [06:10<00:00, 21.55it/s]\n",
            "2023-02-28 16:57:34   Calculating recalls\n",
            "2023-02-28 16:57:36   Epoch 00 in 1:16:06, < val - #q: 7993; #db: 8015 >: R@1: 80.1, R@5: 89.4\n",
            "100%|███████████████████████████████████████████████████████| 10000/10000 [1:10:05<00:00,  2.38it/s]\n",
            "2023-02-28 18:07:43   Epoch 01 in 1:10:06, loss = 1.8135\n",
            "2023-02-28 18:07:43   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:26<00:00, 18.94it/s]\n",
            "2023-02-28 18:08:09   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [06:14<00:00, 21.36it/s]\n",
            "2023-02-28 18:14:24   Calculating recalls\n",
            "2023-02-28 18:14:25   Epoch 01 in 1:16:48, < val - #q: 7993; #db: 8015 >: R@1: 84.1, R@5: 91.8\n",
            "100%|███████████████████████████████████████████████████████| 10000/10000 [1:09:25<00:00,  2.40it/s]\n",
            "2023-02-28 19:23:53   Epoch 02 in 1:09:27, loss = 1.2402\n",
            "2023-02-28 19:23:53   Extracting database descriptors for evaluation/testing\n",
            "100%|█████████████████████████████████████████████████████████████| 501/501 [00:25<00:00, 19.40it/s]\n",
            "2023-02-28 19:24:19   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 7993/7993 [06:15<00:00, 21.31it/s]\n",
            "2023-02-28 19:30:34   Calculating recalls\n",
            "2023-02-28 19:30:35   Epoch 02 in 1:16:09, < val - #q: 7993; #db: 8015 >: R@1: 85.7, R@5: 92.6\n",
            "2023-02-28 19:30:36   The training is over in 3:49:14, now it's test time\n",
            "2023-02-28 19:30:36   Now testing on the test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-02-28 19:30:36   Extracting database descriptors for evaluation/testing\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [01:26<00:00, 19.59it/s]\n",
            "2023-02-28 19:32:03   Extracting queries descriptors for evaluation/testing using batch size 1\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:46<00:00, 21.38it/s]\n",
            "2023-02-28 19:32:50   Calculating recalls\n",
            "2023-02-28 19:32:50   < test - #q: 1000; #db: 27191 >: R@1: 60.0, R@5: 73.2, R@10: 77.3, R@20: 80.9\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [03:27<00:00,  4.81it/s]\n",
            "2023-02-28 19:36:18   test with no warping: R@1: 60.0, R@5: 73.2, R@10: 77.3, R@20: 80.9\n",
            "2023-02-28 19:36:18   test after warping - R@1: 67.0, R@5: 73.2, R@10: 77.3, R@20: 80.9\n",
            "2023-02-28 19:36:18   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/train_geowarp.py --dataset_folder /content/small --groups_num 1 --epochs_num 3 --backbone efficientnet_v2_s\n",
        "#BE CAREFUL: THE MODEL IS ALREADY TRAINED AND IT IS IN geowarp_model"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "UPBUAVhoweDv"
      },
      "source": [
        "## Test on sf-xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "F_dZBhnf22W9"
      },
      "source": [
        "Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 60.0, R@5: 73.3, R@10: 77.3, R@20: 80.9\n",
        "\n",
        "Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 67.1, R@5: 73.3, R@10: 77.3, R@20: 80.9\n",
        "\n",
        "21-04\n",
        "\n",
        "Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 62.5, R@5: 74.9, R@10: 78.8, R@20: 81.7\n",
        "\n",
        "Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 68.6, R@5: 74.9, R@10: 78.8, R@20: 81.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ve-Y-yULwfyX",
        "outputId": "c5c1b0c1-e575-4eb4-f0d8-1a540c4ae1b7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 12:05:59   /content/evalGeowarp.py --dataset_folder /content/small/ --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-04-21 12:05:59   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-21 12:05:59   The outputs are being saved in logs/default/2023-04-21_12-05-59\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 12:06:00   Using efficient net v2s\n",
            "2023-04-21 12:06:00   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 12:06:03   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 12:06:03   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-04-21 12:06:04   Start testing\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:50<00:00,  4.14it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:36<00:00, 27.56it/s]\n",
            "2023-04-21 12:13:31   Start re-ranking\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [06:41<00:00,  2.49it/s]\n",
            "2023-04-21 12:20:13   Test without warping: < test - #q: 1000; #db: 27191 >: R@1: 62.5, R@5: 74.9, R@10: 78.8, R@20: 81.7\n",
            "2023-04-21 12:20:13     Test after warping: < test - #q: 1000; #db: 27191 >: R@1: 68.6, R@5: 74.9, R@10: 78.8, R@20: 81.7\n",
            "2023-04-21 12:20:13   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/small/ --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "uw43U9Z7wf-u"
      },
      "source": [
        "## Test on tokyo-xs"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "EvGEwTh826J1"
      },
      "source": [
        "Test without warping: < test - #q: 315; #db: 12771 >: R@1: 78.4, R@5: 90.5, R@10: 93.7, R@20: 96.8\n",
        "\n",
        "Test after warping: < test - #q: 315; #db: 12771 >: R@1: 84.1, R@5: 90.5, R@10: 93.7, R@20: 96.8\n",
        "\n",
        "21-04\n",
        "\n",
        " Test without warping: < test - #q: 315; #db: 12771 >: R@1: 83.2, R@5: 92.1, R@10: 95.2, R@20: 97.5\n",
        " \n",
        " Test after warping: < test - #q: 315; #db: 12771 >: R@1: 85.4, R@5: 92.1, R@10: 95.2, R@20: 97.5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rXKRou2BwiH-",
        "outputId": "e34e9527-1988-4819-db06-1c37d4cecb1c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 12:20:16   /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s\n",
            "2023-04-21 12:20:16   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 12:20:16   The outputs are being saved in logs/default/2023-04-21_12-20-16\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 12:20:16   Using efficient net v2s\n",
            "2023-04-21 12:20:16   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 12:20:19   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 12:20:19   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-04-21 12:20:19   Start testing\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:52<00:00,  3.43it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:10<00:00, 29.03it/s]\n",
            "2023-04-21 12:24:23   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [02:14<00:00,  2.35it/s]\n",
            "2023-04-21 12:26:37   Test without warping: < test - #q: 315; #db: 12771 >: R@1: 83.2, R@5: 92.1, R@10: 95.2, R@20: 97.5\n",
            "2023-04-21 12:26:37     Test after warping: < test - #q: 315; #db: 12771 >: R@1: 85.4, R@5: 92.1, R@10: 95.2, R@20: 97.5\n",
            "2023-04-21 12:26:37   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo_xs --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s #file in logs.zip"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "HptBlKsbwiSf"
      },
      "source": [
        "## Test on tokyo night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "MAjTVtfT29bK"
      },
      "source": [
        "Test without warping: < test - #q: 105; #db: 12771 >: R@1: 68.6, R@5: 83.8, R@10: 90.5, R@20: 94.3\n",
        "\n",
        "Test after warping: < test - #q: 105; #db: 12771 >: R@1: 75.2, R@5: 83.8, R@10: 90.5, R@20: 94.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K605qKNAwj8f",
        "outputId": "31ce91b7-bc8c-43db-e0be-305721123350"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 12:26:42   /content/evalGeowarp.py --dataset_folder /content/tokyo-night --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_test True --night_brightness 0.2\n",
            "2023-04-21 12:26:42   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/eff2vs_geowarp/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 12:26:42   The outputs are being saved in logs/default/2023-04-21_12-26-42\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 12:26:42   Using efficient net v2s\n",
            "2023-04-21 12:26:42   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 12:26:45   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 12:26:45   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-04-21 12:26:45   Start testing\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:56<00:00,  3.38it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:04<00:00, 24.79it/s]\n",
            "2023-04-21 12:30:46   Start re-ranking\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:44<00:00,  2.34it/s]\n",
            "2023-04-21 12:31:31   Test without warping: < test - #q: 105; #db: 12771 >: R@1: 68.6, R@5: 83.8, R@10: 90.5, R@20: 94.3\n",
            "2023-04-21 12:31:31     Test after warping: < test - #q: 105; #db: 12771 >: R@1: 75.2, R@5: 83.8, R@10: 90.5, R@20: 94.3\n",
            "2023-04-21 12:31:31   Experiment finished (without any errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/evalGeowarp.py --dataset_folder /content/tokyo-night --resume_model /content/eff2vs_geowarp/best_model.pth --backbone efficientnet_v2_s --night_test True --night_brightness 0.2 #file in logs.zip "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "2QIYv_qT8NX1"
      },
      "source": [
        "# TRAIN EFFNETV2s + GeoWarp + GRL"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-0fwbSmv8THg",
        "outputId": "d66c06a1-16a0-404c-9c1d-73fa8569fa48"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-03-02 17:10:56   /content/train_geowarp_grl.py --dataset_folder /content/small --groups_num 1 --iterations_per_epoch 10 --epochs_num 3 --backbone efficientnet_v2_s --grl_param 0.3 --source_dir /content/small --target_dir /content/night_target --num_workers 2\n",
            "2023-03-02 17:10:56   Arguments: Namespace(L=2, M=10, N=5, alpha=30, augmentation_device='cuda', backbone='efficientnet_v2_s', batch_size=32, brightness=0.7, channels=[225, 128, 128, 64, 64, 64, 64], classifiers_lr=0.01, consistency_w=0.1, contrast=0.7, dataset_folder='/content/small', device='cuda', epochs_num=3, fc_output_dim=512, features_wise_w=10, grl_param=0.3, groups_num=1, hue=0.5, infer_batch_size=16, iterations_per_epoch=10, k=0.6, kernel_sizes=[7, 5, 5, 5, 5, 5], loss_function='cosface', lr=1e-05, min_images_per_class=10, multi_scale=False, multi_scale_method='avg', night_brightness=0.1, night_test=False, num_reranked_preds=5, num_workers=2, optim='adam', positive_dist_threshold=25, qp_threshold=1.2, random_resized_crop=0.5, resize=[480, 640], resume_model=None, resume_train=None, saturation=0.7, save_dir='default', seed=0, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], source_dir='/content/small', ss_w=1, target_dir='/content/night_target', test_method='hard_resize', test_set_folder='/content/small/test', train_set_folder='/content/small/train', use_amp16=False, val_set_folder='/content/small/val', wd=None)\n",
            "2023-03-02 17:10:56   The outputs are being saved in logs/default/2023-03-02_17-10-56\n",
            "2023-03-02 17:10:56   Gradient Reversal Layer is enabled with parameter = 0.3\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-03-02 17:10:57   Using efficient net v2s\n",
            "2023-03-02 17:10:57   Model uses weights IMAGENET1K_V1\n",
            "2023-03-02 17:10:57   Train last two layers of EfficientNet, freeze the previous ones\n",
            "2023-03-02 17:10:57   WARNING: --resume_fe is set to None, meaning that the Feature Extractor is not initialized!\n",
            "2023-03-02 17:11:00   There are 1 GPUs and 2 CPUs.\n",
            "2023-03-02 17:11:00   Using cached dataset cache/small_M10_N5_mipc10.torch\n",
            "2023-03-02 17:11:00   GrlDataset has 3 domain classes\n",
            "2023-03-02 17:11:00   Using cosface loss function.\n",
            "2023-03-02 17:11:00   Using 1 groups\n",
            "2023-03-02 17:11:00   The 1 groups have respectively the following number of classes [5965]\n",
            "2023-03-02 17:11:00   The 1 groups have respectively the following number of images [59650]\n",
            "2023-03-02 17:11:00   Validation set: < val - #q: 7993; #db: 8015 >\n",
            "2023-03-02 17:11:00   Test set: < test - #q: 1000; #db: 27191 >\n",
            "2023-03-02 17:11:00   Start training ...\n",
            "2023-03-02 17:11:00   There are 5965 classes for the first group, each epoch has 10 iterations with batch_size 32, therefore the model sees each class (on average) 0.1 times per epoch\n",
            "  0%|                                                                        | 0/10 [00:10<?, ?it/s]\n",
            "2023-03-02 17:11:12   \n",
            "Traceback (most recent call last):\n",
            "  File \"/content/train_geowarp_grl.py\", line 234, in <module>\n",
            "    domain_adapt_output = model(domain_adapt_images, [images, \"global\"], force_grl=True)\n",
            "NameError: name 'images' is not defined\n",
            "\n",
            "2023-03-02 17:11:12   Experiment finished (with some errors)\n"
          ]
        }
      ],
      "source": [
        "!python /content/train_geowarp_grl.py --dataset_folder /content/small --groups_num 1 --iterations_per_epoch 10 --epochs_num 3 --backbone efficientnet_v2_s --grl_param 0.3 --source_dir /content/small --target_dir /content/night_target  --num_workers 2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Ipn36DpSJXWH"
      },
      "source": [
        "# Ensemble (backbone=effNV2s, GRL and GeoWarp)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "C3V-8RGl3HBe"
      },
      "source": [
        "R@1: 65.3, R@5: 73.8, R@10: 77.1, R@20: 79.9\n",
        "\n",
        "21 04\n",
        "\n",
        "test - #q: 1000; #db: 27191 >: R@1: 67.4, R@5: 75.1, R@10: 78.2, R@20: 80.8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9GIn-kLYJh7W",
        "outputId": "d8801163-ce9b-403b-e9d4-a66949b0af0a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 12:42:14   eval_ensemble.py --dataset_folder /content/small/ --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-04-21 12:42:14   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/small/test')\n",
            "2023-04-21 12:42:14   The outputs are being saved in logs/default/2023-04-21_12-42-14\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 12:42:15   Using efficient net v2s\n",
            "2023-04-21 12:42:15   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 12:42:16   Using efficient net v2s\n",
            "2023-04-21 12:42:16   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 12:42:17   Using efficient net v2s\n",
            "2023-04-21 12:42:17   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 12:42:20   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 12:42:20   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-04-21 12:42:20   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:29<00:00,  4.36it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:32<00:00, 30.99it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:48<00:00,  4.16it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:35<00:00, 28.56it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [06:43<00:00,  2.48it/s]\n",
            "2023-04-21 13:03:32   < test - #q: 1000; #db: 27191 >: R@1: 67.4, R@5: 75.1, R@10: 78.2, R@20: 80.8\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GbmokQTLObun",
        "outputId": "903b7191-93c7-4ef0-9733-a70401b14c93"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-07 09:28:35   eval_ensemble.py --dataset_folder /content/small/ --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-07 09:28:35   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/small/test')\n",
            "2023-05-07 09:28:35   The outputs are being saved in logs/default/2023-05-07_09-28-35\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-07 09:28:36   Using efficient net v2s\n",
            "2023-05-07 09:28:36   Model uses weights IMAGENET1K_V1\n",
            "2023-05-07 09:28:37   Using efficient net v2s\n",
            "2023-05-07 09:28:37   Model uses weights IMAGENET1K_V1\n",
            "2023-05-07 09:28:38   Using efficient net v2s\n",
            "2023-05-07 09:28:38   Model uses weights IMAGENET1K_V1\n",
            "2023-05-07 09:28:40   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-07 09:28:40   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-07 09:28:40   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:18<00:00,  4.49it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:33<00:00, 29.50it/s]\n",
            "GRL: [1.1033123 1.150476  1.1570008 1.1674622 1.1719211 1.2285672 1.2371438\n",
            " 1.2434522 1.2564212 1.2812889 1.3198149 1.3286748 1.3567135 1.3581052\n",
            " 1.373511  1.3800602 1.3821516 1.3861392 1.4025122 1.4030298], [   31    35    33    34    29    25    32 13523    39    26    37   321\n",
            "    38    12  7465  4867 19532  8827  2954  1694]\n",
            "(1000, 20)\n",
            "(1000, 20)\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:39<00:00,  4.26it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:35<00:00, 28.19it/s]\n",
            "PRE RANK: [1.1541138 1.1753956 1.1981382 1.2057875 1.2099955 1.2102652 1.2486494\n",
            " 1.251767  1.2564073 1.2759082 1.277071  1.3147771 1.3167626 1.3377028\n",
            " 1.3441069 1.3444619 1.3558141 1.3608794 1.3681107 1.371563 ], [   35    31    33    29    34    25    26    39    37    32 13523    95\n",
            "  8827    12  2951   321 14136  2954  2952    38]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [06:37<00:00,  2.51it/s]\n",
            "POST RANK: [1.2099955 1.2057875 1.1753956 1.1541138 1.1981382 1.2102652 1.2486494\n",
            " 1.251767  1.2564073 1.2759082 1.277071  1.3147771 1.3167626 1.3377028\n",
            " 1.3441069 1.3444619 1.3558141 1.3608794 1.3681107 1.371563 ], [   34    29    31    35    33    25    26    39    37    32 13523    95\n",
            "  8827    12  2951   321 14136  2954  2952    38]\n",
            "2023-05-07 09:49:27   < test - #q: 1000; #db: 27191 >: R@1: 64.0, R@5: 73.9, R@10: 78.1, R@20: 80.8\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "qmBtJ8g03JSm"
      },
      "source": [
        "R@1: 84.1, R@5: 91.1, R@10: 94.0, R@20: 95.6\n",
        "\n",
        "21 04\n",
        "\n",
        "R@1: 83.8, R@5: 93.0, R@10: 95.2, R@20: 97.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MVH3Ndc6N8vb",
        "outputId": "27059363-4446-4afe-c7aa-3ab1c7428907"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 13:03:36   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-04-21 13:03:36   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/tokyo_xs/test')\n",
            "2023-04-21 13:03:36   The outputs are being saved in logs/default/2023-04-21_13-03-36\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 13:03:37   Using efficient net v2s\n",
            "2023-04-21 13:03:37   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 13:03:38   Using efficient net v2s\n",
            "2023-04-21 13:03:38   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 13:03:39   Using efficient net v2s\n",
            "2023-04-21 13:03:39   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 13:03:42   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 13:03:42   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-04-21 13:03:42   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:45<00:00,  3.54it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:09<00:00, 33.98it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:51<00:00,  3.45it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:10<00:00, 30.24it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [02:13<00:00,  2.35it/s]\n",
            "2023-04-21 13:13:54   < test - #q: 315; #db: 12771 >: R@1: 83.8, R@5: 93.0, R@10: 95.2, R@20: 97.1\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "mRsjjJM63Nc2"
      },
      "source": [
        "R@1: 76.2, R@5: 86.7, R@10: 87.6, R@20: 93.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZGrgQe-TN9Eb",
        "outputId": "73b30a53-be98-4418-8b4a-4d9a8e0fef30"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-21 13:13:58   eval_ensemble.py --dataset_folder /content/tokyo-night/ --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2\n",
            "2023-04-21 13:13:58   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/tokyo-night/test')\n",
            "2023-04-21 13:13:58   The outputs are being saved in logs/default/2023-04-21_13-13-58\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-04-21 13:13:59   Using efficient net v2s\n",
            "2023-04-21 13:13:59   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 13:14:00   Using efficient net v2s\n",
            "2023-04-21 13:14:00   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 13:14:01   Using efficient net v2s\n",
            "2023-04-21 13:14:01   Model uses weights IMAGENET1K_V1\n",
            "2023-04-21 13:14:03   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-21 13:14:03   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-04-21 13:14:03   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:48<00:00,  3.49it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 28.44it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:54<00:00,  3.41it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:04<00:00, 22.71it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:44<00:00,  2.35it/s]\n",
            "2023-04-21 13:22:40   < test - #q: 105; #db: 12771 >: R@1: 76.2, R@5: 86.7, R@10: 87.6, R@20: 93.3\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "N1Y1kiT_vSQk"
      },
      "source": [
        "# Testing the two final ensembler techniques"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "di9lI_1Zvdr7"
      },
      "source": [
        "## Technique #1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1nZM6bLHEvfW",
        "outputId": "f691d553-0026-4d27-e4f3-74a1e2882f95"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-16 08:36:07   eval_ensemble.py --dataset_folder /content/small/ --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-16 08:36:07   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=False, test_set_folder='/content/small/test')\n",
            "2023-05-16 08:36:07   The outputs are being saved in logs/default/2023-05-16_08-36-07\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-16 08:36:08   Using efficient net v2s\n",
            "2023-05-16 08:36:08   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 08:36:10   Using efficient net v2s\n",
            "2023-05-16 08:36:10   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 08:36:11   Using efficient net v2s\n",
            "2023-05-16 08:36:11   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 08:36:14   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-16 08:36:14   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-16 08:36:15   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:31<00:00,  4.35it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:33<00:00, 30.25it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [06:51<00:00,  4.13it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:35<00:00, 27.95it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [06:44<00:00,  2.47it/s]\n",
            "False\n",
            "sono true\n",
            "2023-05-16 08:57:32   < test - #q: 1000; #db: 27191 >: R@1: 67.4, R@5: 76.0, R@10: 79.2, R@20: 82.4\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X_i-zrnwvroH",
        "outputId": "67461d7e-3f49-4285-e36b-0e1d51109d8f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-16 08:57:36   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-16 08:57:36   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=False, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-16 08:57:36   The outputs are being saved in logs/default/2023-05-16_08-57-36\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-16 08:57:36   Using efficient net v2s\n",
            "2023-05-16 08:57:36   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 08:57:37   Using efficient net v2s\n",
            "2023-05-16 08:57:37   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 08:57:38   Using efficient net v2s\n",
            "2023-05-16 08:57:38   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 08:57:41   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-16 08:57:41   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-16 08:57:42   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:45<00:00,  3.54it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:09<00:00, 33.11it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:51<00:00,  3.45it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:10<00:00, 30.39it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [02:14<00:00,  2.34it/s]\n",
            "False\n",
            "sono true\n",
            "2023-05-16 09:07:55   < test - #q: 315; #db: 12771 >: R@1: 83.8, R@5: 93.0, R@10: 95.9, R@20: 97.5\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qHJFvPmQsSUv",
        "outputId": "7b40b010-7c3a-45ff-ea55-fe58b94b5d37"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-16 09:07:59   eval_ensemble.py --dataset_folder /content/tokyo-night/ --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2\n",
            "2023-05-16 09:07:59   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=False, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-16 09:07:59   The outputs are being saved in logs/default/2023-05-16_09-07-59\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-16 09:07:59   Using efficient net v2s\n",
            "2023-05-16 09:07:59   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 09:08:00   Using efficient net v2s\n",
            "2023-05-16 09:08:00   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 09:08:01   Using efficient net v2s\n",
            "2023-05-16 09:08:01   Model uses weights IMAGENET1K_V1\n",
            "2023-05-16 09:08:04   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-16 09:08:04   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-16 09:08:04   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:561: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:47<00:00,  3.52it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 28.91it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [03:53<00:00,  3.42it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:04<00:00, 22.12it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:45<00:00,  2.29it/s]\n",
            "False\n",
            "sono true\n",
            "2023-05-16 09:16:39   < test - #q: 105; #db: 12771 >: R@1: 76.2, R@5: 86.7, R@10: 91.4, R@20: 94.3\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2  "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "JsaSu5hF7v_L"
      },
      "source": [
        "## AVG"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wbiMzFXu8DKs",
        "outputId": "a78bbae7-bedc-4780-f3bc-cb711db7efdf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 10:34:54   eval_ensemble.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 10:34:54   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/small/test')\n",
            "2023-05-22 10:34:54   The outputs are being saved in logs/default/2023-05-22_10-34-54\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 10:34:54   Using efficient net v2s\n",
            "2023-05-22 10:34:54   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 10:34:55   Using efficient net v2s\n",
            "2023-05-22 10:34:55   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 10:34:56   Using efficient net v2s\n",
            "2023-05-22 10:34:56   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 10:34:59   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 10:34:59   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 10:34:59   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 10:34:59   Test with multi-scale, the multi-scale method is: avg\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:20<00:00,  1.98it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:23<00:00, 42.56it/s]\n",
            "GRL: [1.0566031 1.1082726 1.1210393 1.1266681 1.1767209 1.1789248 1.1959263\n",
            " 1.2355698 1.2386187 1.2665728 1.2930148 1.3015004 1.3503474 1.3532035\n",
            " 1.3545408 1.3610137 1.3669705 1.3793987 1.3849285 1.3867146], [   31    35    33    34    32    29    25    37    39    26    38 13523\n",
            "  2954  8827   110   321  1694 16121  4867   964]\n",
            "2023-05-22 10:49:44   Test with multi-scale, the multi-scale method is: avg\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:52<00:00,  1.91it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:24<00:00, 40.93it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:09<00:00,  7.73it/s]\n",
            "2023-05-22 11:07:10   < test - #q: 1000; #db: 27191 >: R@1: 62.2, R@5: 73.6, R@10: 78.2, R@20: 81.4\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tuGx88ZA8De4",
        "outputId": "e45496d8-8842-4e2c-e008-2cda8b483ae2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 11:07:48   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 11:07:48   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-22 11:07:48   The outputs are being saved in logs/default/2023-05-22_11-07-48\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 11:07:49   Using efficient net v2s\n",
            "2023-05-22 11:07:49   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:07:49   Using efficient net v2s\n",
            "2023-05-22 11:07:49   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:07:50   Using efficient net v2s\n",
            "2023-05-22 11:07:50   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:07:53   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 11:07:53   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 11:07:53   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 11:07:53   Test with multi-scale, the multi-scale method is: avg\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:43<00:00,  1.98it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:07<00:00, 43.74it/s]\n",
            "GRL: [0.935794  1.0522698 1.0805287 1.1249816 1.1427879 1.1613271 1.1670668\n",
            " 1.1812533 1.1861949 1.2068074 1.2105196 1.2188625 1.2242143 1.227111\n",
            " 1.228023  1.239854  1.2430248 1.2455132 1.2468957 1.247087 ], [  685   703   714   713   740   627  7330   699 12340  1542  4726 10390\n",
            "   553  4722   594 12392  6225  2477   463  6171]\n",
            "2023-05-22 11:14:44   Test with multi-scale, the multi-scale method is: avg\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:57<00:00,  1.91it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 39.31it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:40<00:00,  7.79it/s]\n",
            "2023-05-22 11:22:30   < test - #q: 315; #db: 12771 >: R@1: 81.6, R@5: 93.0, R@10: 96.5, R@20: 97.8\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pelvNU9i8Du1",
        "outputId": "149eb697-d292-4f3d-c3f0-ea03b576113c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 11:25:01   eval_ensemble.py --dataset_folder /content/tokyo-night/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2\n",
            "2023-05-22 11:25:01   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-22 11:25:01   The outputs are being saved in logs/default/2023-05-22_11-25-01\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 11:25:02   Using efficient net v2s\n",
            "2023-05-22 11:25:02   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:25:03   Using efficient net v2s\n",
            "2023-05-22 11:25:03   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:25:04   Using efficient net v2s\n",
            "2023-05-22 11:25:04   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:25:06   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 11:25:06   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 11:25:06   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 11:25:06   Test with multi-scale, the multi-scale method is: avg\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:51<00:00,  1.94it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.11it/s]\n",
            "GRL: [1.3194516 1.3758029 1.3917913 1.4247792 1.434499  1.4347239 1.4411465\n",
            " 1.458203  1.4621675 1.4666859 1.4673239 1.4690382 1.4706559 1.4715996\n",
            " 1.4736452 1.4742923 1.4744022 1.4748449 1.4780047 1.4805776], [ 3571  1243 10492  3058  3214 10513  1096 11807  9052 10522 11489  8675\n",
            " 10436   879  5479  2121  3052   455   714  8325]\n",
            "2023-05-22 11:32:01   Test with multi-scale, the multi-scale method is: avg\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:07<00:00,  1.87it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 36.84it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:13<00:00,  7.63it/s]\n",
            "2023-05-22 11:39:25   < test - #q: 105; #db: 12771 >: R@1: 69.5, R@5: 86.7, R@10: 90.5, R@20: 94.3\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "gqK-5mUK7xjX"
      },
      "source": [
        "## SUM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F73lB5Jb-Ehu",
        "outputId": "8c9e5f93-0290-4121-e3b3-eade202effc0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 11:41:21   eval_ensemble.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 11:41:21   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/small/test')\n",
            "2023-05-22 11:41:21   The outputs are being saved in logs/default/2023-05-22_11-41-21\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 11:41:22   Using efficient net v2s\n",
            "2023-05-22 11:41:22   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:41:23   Using efficient net v2s\n",
            "2023-05-22 11:41:23   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:41:24   Using efficient net v2s\n",
            "2023-05-22 11:41:24   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 11:41:26   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 11:41:26   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 11:41:26   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 11:41:27   Test with multi-scale, the multi-scale method is: sum\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:17<00:00,  1.98it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:23<00:00, 42.15it/s]\n",
            "GRL: [20.81928  21.066587 21.1134   21.13028  21.27381  21.46241  21.509005\n",
            " 21.848972 21.863846 21.90829  22.02586  22.044386 22.121933 22.191021\n",
            " 22.25121  22.274809 22.300226 22.315832 22.324942 22.325613], [   31    34    33    35    32    29    25    39    26    37 13523 18898\n",
            "    38 10543  2954   110   964   321  2951  8827]\n",
            "2023-05-22 11:56:08   Test with multi-scale, the multi-scale method is: sum\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:54<00:00,  1.90it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:24<00:00, 41.13it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:11<00:00,  7.60it/s]\n",
            "2023-05-22 12:13:38   < test - #q: 1000; #db: 27191 >: R@1: 61.1, R@5: 73.9, R@10: 77.9, R@20: 81.3\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QrR2GFaE-EYJ",
        "outputId": "a1b8e2b2-a0b5-40f5-88c8-455f986a208f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 12:19:54   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 12:19:54   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-22 12:19:54   The outputs are being saved in logs/default/2023-05-22_12-19-54\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 12:19:55   Using efficient net v2s\n",
            "2023-05-22 12:19:55   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 12:19:55   Using efficient net v2s\n",
            "2023-05-22 12:19:55   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 12:19:56   Using efficient net v2s\n",
            "2023-05-22 12:19:56   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 12:19:59   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 12:19:59   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 12:19:59   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 12:19:59   Test with multi-scale, the multi-scale method is: sum\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:49<00:00,  1.95it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:07<00:00, 43.29it/s]\n",
            "GRL: [19.34132  20.278208 20.377666 20.447933 20.566923 20.917501 20.941057\n",
            " 20.950035 21.10931  21.168331 21.172089 21.178984 21.20019  21.208769\n",
            " 21.209566 21.220835 21.224606 21.257986 21.325829 21.336926], [  685   714   703   740  6225   713 12340  7330   764 12595   553   594\n",
            "  1725   627   835  2477  4726   699 10390  1542]\n",
            "2023-05-22 12:26:56   Test with multi-scale, the multi-scale method is: sum\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:58<00:00,  1.91it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 38.84it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:41<00:00,  7.63it/s]\n",
            "2023-05-22 12:34:44   < test - #q: 315; #db: 12771 >: R@1: 74.9, R@5: 89.8, R@10: 92.1, R@20: 95.9\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IPS5kR98-ELF",
        "outputId": "bd577598-fdf5-4df3-d3c0-d2b8a6f66247"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 12:38:27   eval_ensemble.py --dataset_folder /content/tokyo-night/ --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2\n",
            "2023-05-22 12:38:27   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-22 12:38:27   The outputs are being saved in logs/default/2023-05-22_12-38-27\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 12:38:28   Using efficient net v2s\n",
            "2023-05-22 12:38:28   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 12:38:28   Using efficient net v2s\n",
            "2023-05-22 12:38:28   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 12:38:29   Using efficient net v2s\n",
            "2023-05-22 12:38:29   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 12:38:32   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 12:38:32   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 12:38:32   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 12:38:32   Test with multi-scale, the multi-scale method is: sum\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:52<00:00,  1.94it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.44it/s]\n",
            "GRL: [21.483734 21.91606  21.926107 22.03066  22.077948 22.08957  22.153023\n",
            " 22.212494 22.213795 22.215094 22.224848 22.26927  22.277193 22.289375\n",
            " 22.341854 22.431076 22.453701 22.483105 22.483501 22.486364], [ 3571  1522 10492  8675  9150  8737   455 10398  2121  3058   714  3524\n",
            "  1243  3683  3214  8738  5144  5133 11303 12171]\n",
            "2023-05-22 12:45:28   Test with multi-scale, the multi-scale method is: sum\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:11<00:00,  1.85it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 33.42it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:14<00:00,  7.24it/s]\n",
            "2023-05-22 12:52:56   < test - #q: 105; #db: 12771 >: R@1: 67.6, R@5: 81.9, R@10: 87.6, R@20: 94.3\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "KHJ47qrN7_1c"
      },
      "source": [
        "## MAX"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZFlxrZBy-SA1",
        "outputId": "287d786a-bcf9-4295-a2a0-d0cad70b45e5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 13:59:28   eval_ensemble.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 13:59:28   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/small/test')\n",
            "2023-05-22 13:59:28   The outputs are being saved in logs/default/2023-05-22_13-59-28\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 13:59:29   Using efficient net v2s\n",
            "2023-05-22 13:59:29   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 13:59:30   Using efficient net v2s\n",
            "2023-05-22 13:59:30   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 13:59:30   Using efficient net v2s\n",
            "2023-05-22 13:59:30   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 13:59:33   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 13:59:33   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 13:59:33   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 13:59:34   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:39<00:00,  1.93it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:23<00:00, 42.22it/s]\n",
            "GRL: [1.0750091 1.105794  1.142057  1.1491336 1.1982573 1.2041144 1.2147048\n",
            " 1.240677  1.2434177 1.2997102 1.3016529 1.3246946 1.3656375 1.3656429\n",
            " 1.3739171 1.3777853 1.3822582 1.3836815 1.3842924 1.3892176], [   31    35    33    34    29    25    32    39    37    38    26 13523\n",
            "  2954   110 16121  1694  8827   321  4867  9255]\n",
            "2023-05-22 14:14:37   Test with multi-scale, the multi-scale method is: max\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [15:09<00:00,  1.87it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:26<00:00, 37.20it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:19<00:00,  7.16it/s]\n",
            "2023-05-22 14:32:34   < test - #q: 1000; #db: 27191 >: R@1: 60.7, R@5: 72.5, R@10: 76.4, R@20: 80.0\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_vu0xHd3-R1n",
        "outputId": "c41c86ac-c38b-407b-ede6-d13411c27d72"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 15:03:08   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 15:03:08   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-22 15:03:08   The outputs are being saved in logs/default/2023-05-22_15-03-08\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 15:03:08   Using efficient net v2s\n",
            "2023-05-22 15:03:08   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 15:03:09   Using efficient net v2s\n",
            "2023-05-22 15:03:09   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 15:03:10   Using efficient net v2s\n",
            "2023-05-22 15:03:10   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 15:03:13   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 15:03:13   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 15:03:13   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 15:03:13   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:55<00:00,  1.92it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:07<00:00, 39.94it/s]\n",
            "GRL: [1.0502496 1.113689  1.1741686 1.1842535 1.1848673 1.2011849 1.2465967\n",
            " 1.247319  1.2513933 1.256136  1.2566972 1.261983  1.2704458 1.2779176\n",
            " 1.2810608 1.2863259 1.2909162 1.2956309 1.2966197 1.3046079], [  685   703   627   713   714   699  1542  4726   740  7330 10390 12340\n",
            "  4722   553   463  6171   594  2153 11438  5091]\n",
            "2023-05-22 15:10:17   Test with multi-scale, the multi-scale method is: max\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:12<00:00,  1.85it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:09<00:00, 34.17it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:44<00:00,  7.03it/s]\n",
            "2023-05-22 15:18:23   < test - #q: 315; #db: 12771 >: R@1: 80.3, R@5: 93.0, R@10: 94.9, R@20: 97.8\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "924RctDj-RmS"
      },
      "outputs": [],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "_SY3FK7C70PG"
      },
      "source": [
        "## MIN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "yC57AwtW-cph",
        "outputId": "a1c25efc-3fc4-4516-b669-68a3077a0e64"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 15:18:32   eval_ensemble.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 15:18:32   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/small/test')\n",
            "2023-05-22 15:18:32   The outputs are being saved in logs/default/2023-05-22_15-18-32\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-22 15:18:33   Using efficient net v2s\n",
            "2023-05-22 15:18:33   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 15:18:34   Using efficient net v2s\n",
            "2023-05-22 15:18:34   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 15:18:35   Using efficient net v2s\n",
            "2023-05-22 15:18:35   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 15:18:37   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-22 15:18:37   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 15:18:37   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 15:18:38   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:54<00:00,  1.90it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:26<00:00, 37.69it/s]\n",
            "GRL: [1.1088043 1.1519785 1.1560986 1.1643776 1.2299229 1.2383301 1.2644205\n",
            " 1.2732866 1.2940571 1.3125322 1.3239781 1.3788006 1.3946166 1.3977673\n",
            " 1.398977  1.4025424 1.4100957 1.4245229 1.4297608 1.4307431], [   31    33    35    34    32    29    25    37    39    38    26 13523\n",
            "  8827  1694   110  2954   321  7465 14389  4867]\n",
            "2023-05-22 15:33:59   Test with multi-scale, the multi-scale method is: min\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:55<00:00,  1.90it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:25<00:00, 39.94it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:11<00:00,  7.58it/s]\n",
            "2023-05-22 15:51:32   < test - #q: 1000; #db: 27191 >: R@1: 61.2, R@5: 73.0, R@10: 76.3, R@20: 80.6\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gNb-nxQc-ciA",
        "outputId": "517b8e18-e86b-421a-a2d9-06c59c75df80"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-22 16:17:56   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 16:17:56   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-22 16:17:56   The outputs are being saved in logs/default/2023-05-22_16-17-56\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:04<00:00, 17.8MB/s]\n",
            "2023-05-22 16:18:03   Using efficient net v2s\n",
            "2023-05-22 16:18:03   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 16:18:04   Using efficient net v2s\n",
            "2023-05-22 16:18:04   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 16:18:04   Using efficient net v2s\n",
            "2023-05-22 16:18:04   Model uses weights IMAGENET1K_V1\n",
            "2023-05-22 16:18:12   There are 1 GPUs and 2 CPUs.\n",
            "2023-05-22 16:18:12   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-22 16:18:12   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-22 16:18:13   Test with multi-scale, the multi-scale method is: min\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:560: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            " 74%|████████████████████████████████████████████▉                | 588/799 [29:06<10:18,  2.93s/it]"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1ZBhmiWG-cZ5"
      },
      "outputs": [],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "7UweJHrQv5sg"
      },
      "source": [
        "## Technique #2"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "vcRazsxeVo6-"
      },
      "source": [
        "### AVG"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "MSy2xBQMWttG"
      },
      "source": [
        "19-05-2023:\n",
        "\n",
        "sf-xs: test - #q: 1000; #db: 27191 >: R@1: 68.6, R@5: 75.0, R@10: 79.0, R@20: 81.7\n",
        "\n",
        "tokyo_xs: test - #q: 1000; #db: 27191 >: R@1: 68.6, R@5: 75.0, R@10: 79.0, R@20: 81.7\n",
        "\n",
        "tokyo_night: test - #q: 1000; #db: 27191 >: R@1: 68.6, R@5: 75.0, R@10: 79.0, R@20: 81.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JAq2X3AXv8FP",
        "outputId": "1ddc80c6-12b4-463f-9ba1-7c1dfc7b2e61"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 09:36:20   eval_ensemble.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True\n",
            "2023-05-19 09:36:20   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/small/test')\n",
            "2023-05-19 09:36:20   The outputs are being saved in logs/default/2023-05-19_09-36-20\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/efficientnet_v2_s-dd5fe13b.pth\" to /root/.cache/torch/hub/checkpoints/efficientnet_v2_s-dd5fe13b.pth\n",
            "100% 82.7M/82.7M [00:00<00:00, 455MB/s]\n",
            "2023-05-19 09:36:21   Using efficient net v2s\n",
            "2023-05-19 09:36:21   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 09:36:22   Using efficient net v2s\n",
            "2023-05-19 09:36:22   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 09:36:23   Using efficient net v2s\n",
            "2023-05-19 09:36:23   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 09:36:28   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 09:36:28   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 09:36:28   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 09:36:29   Test with multi-scale, the multi-scale method is: avg\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:28<00:00,  1.96it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:24<00:00, 41.14it/s]\n",
            "GRL: [1.0566031 1.1082726 1.1210393 1.1266681 1.1767209 1.1789248 1.1959263\n",
            " 1.2355698 1.2386187 1.2665728 1.2930148 1.3015004 1.3503474 1.3532035\n",
            " 1.3545408 1.3610137 1.3669705 1.3793987 1.3849285 1.3867146], [   31    35    33    34    32    29    25    37    39    26    38 13523\n",
            "  2954  8827   110   321  1694 16121  4867   964]\n",
            "2023-05-19 09:51:22   Test with multi-scale, the multi-scale method is: avg\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:45<00:00,  1.92it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:24<00:00, 41.48it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:09<00:00,  7.70it/s]\n",
            "2023-05-19 10:08:42   < test - #q: 1000; #db: 27191 >: R@1: 67.4, R@5: 72.9, R@10: 78.1, R@20: 80.8\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gCydni1Wv8RV",
        "outputId": "fb353549-aeed-4b47-a8ac-3198af7ce2e9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 10:10:51   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True\n",
            "2023-05-19 10:10:51   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-19 10:10:51   The outputs are being saved in logs/default/2023-05-19_10-10-51\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 10:10:52   Using efficient net v2s\n",
            "2023-05-19 10:10:52   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 10:10:53   Using efficient net v2s\n",
            "2023-05-19 10:10:53   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 10:10:54   Using efficient net v2s\n",
            "2023-05-19 10:10:54   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 10:10:56   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 10:10:56   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 10:10:56   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 10:10:56   Test with multi-scale, the multi-scale method is: avg\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:46<00:00,  1.97it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:06<00:00, 45.30it/s]\n",
            "GRL: [0.935794  1.0522698 1.0805287 1.1249816 1.1427879 1.1613271 1.1670668\n",
            " 1.1812533 1.1861949 1.2068074 1.2105196 1.2188625 1.2242143 1.227111\n",
            " 1.228023  1.239854  1.2430248 1.2455132 1.2468957 1.247087 ], [  685   703   714   713   740   627  7330   699 12340  1542  4726 10390\n",
            "   553  4722   594 12392  6225  2477   463  6171]\n",
            "2023-05-19 10:17:50   Test with multi-scale, the multi-scale method is: avg\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:56<00:00,  1.92it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:07<00:00, 40.83it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:39<00:00,  7.89it/s]\n",
            "2023-05-19 10:25:34   < test - #q: 315; #db: 12771 >: R@1: 86.0, R@5: 92.7, R@10: 96.5, R@20: 97.8\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9t5fNqsHv8p0",
        "outputId": "042f5371-1beb-4f0d-fe66-734eb9c97f72"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 10:26:20   eval_ensemble.py --dataset_folder /content/tokyo-night/ --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2 --ensemble_merge_preds True\n",
            "2023-05-19 10:26:20   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='avg', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-19 10:26:20   The outputs are being saved in logs/default/2023-05-19_10-26-20\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 10:26:20   Using efficient net v2s\n",
            "2023-05-19 10:26:20   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 10:26:21   Using efficient net v2s\n",
            "2023-05-19 10:26:21   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 10:26:22   Using efficient net v2s\n",
            "2023-05-19 10:26:22   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 10:26:24   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 10:26:24   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 10:26:24   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 10:26:25   Test with multi-scale, the multi-scale method is: avg\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:49<00:00,  1.95it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.83it/s]\n",
            "GRL: [1.3194516 1.3758029 1.3917913 1.4247792 1.434499  1.4347239 1.4411465\n",
            " 1.458203  1.4621675 1.4666859 1.4673239 1.4690382 1.4706559 1.4715996\n",
            " 1.4736452 1.4742923 1.4744022 1.4748449 1.4780047 1.4805776], [ 3571  1243 10492  3058  3214 10513  1096 11807  9052 10522 11489  8675\n",
            " 10436   879  5479  2121  3052   455   714  8325]\n",
            "2023-05-19 10:33:17   Test with multi-scale, the multi-scale method is: avg\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:00<00:00,  1.90it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.74it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:13<00:00,  7.68it/s]\n",
            "2023-05-19 10:40:34   < test - #q: 105; #db: 12771 >: R@1: 77.1, R@5: 87.6, R@10: 92.4, R@20: 94.3\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --multi_scale --multi_scale_method=avg --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2 --ensemble_merge_preds True"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "_Z69ulF0VyBG"
      },
      "source": [
        "### SUM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i9IfK4nNWEFx"
      },
      "outputs": [],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EA98XYyNWEFy",
        "outputId": "3cbef5fa-a7b4-4c6b-b67a-2c07a7351191"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 11:29:33   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True\n",
            "2023-05-19 11:29:33   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-19 11:29:33   The outputs are being saved in logs/default/2023-05-19_11-29-33\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 11:29:33   Using efficient net v2s\n",
            "2023-05-19 11:29:33   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:29:34   Using efficient net v2s\n",
            "2023-05-19 11:29:34   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:29:35   Using efficient net v2s\n",
            "2023-05-19 11:29:35   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:29:37   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 11:29:37   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 11:29:37   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 11:29:38   Test with multi-scale, the multi-scale method is: sum\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:51<00:00,  1.94it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:07<00:00, 41.27it/s]\n",
            "GRL: [19.34132  20.278208 20.377666 20.447933 20.566923 20.917501 20.941057\n",
            " 20.950035 21.10931  21.168331 21.172089 21.178984 21.20019  21.208769\n",
            " 21.209566 21.220835 21.224606 21.257986 21.325829 21.336926], [  685   714   703   740  6225   713 12340  7330   764 12595   553   594\n",
            "  1725   627   835  2477  4726   699 10390  1542]\n",
            "2023-05-19 11:36:37   Test with multi-scale, the multi-scale method is: sum\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:04<00:00,  1.88it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 36.84it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:42<00:00,  7.44it/s]\n",
            "2023-05-19 11:44:32   < test - #q: 315; #db: 12771 >: R@1: 81.3, R@5: 90.2, R@10: 94.0, R@20: 95.2\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f7qPKU7zWEFy",
        "outputId": "d9ef270a-6a94-4b6d-faaa-dd83e8d14078"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 11:44:36   eval_ensemble.py --dataset_folder /content/tokyo-night/ --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2 --ensemble_merge_preds True\n",
            "2023-05-19 11:44:36   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='sum', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-19 11:44:36   The outputs are being saved in logs/default/2023-05-19_11-44-36\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 11:44:37   Using efficient net v2s\n",
            "2023-05-19 11:44:37   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:44:38   Using efficient net v2s\n",
            "2023-05-19 11:44:38   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:44:39   Using efficient net v2s\n",
            "2023-05-19 11:44:39   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:44:41   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 11:44:41   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 11:44:41   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 11:44:41   Test with multi-scale, the multi-scale method is: sum\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:50<00:00,  1.95it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.42it/s]\n",
            "GRL: [21.483734 21.91606  21.926107 22.03066  22.077948 22.08957  22.153023\n",
            " 22.212494 22.213795 22.215094 22.224848 22.26927  22.277193 22.289375\n",
            " 22.341854 22.431076 22.453701 22.483105 22.483501 22.486364], [ 3571  1522 10492  8675  9150  8737   455 10398  2121  3058   714  3524\n",
            "  1243  3683  3214  8738  5144  5133 11303 12171]\n",
            "2023-05-19 11:51:35   Test with multi-scale, the multi-scale method is: sum\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:04<00:00,  1.88it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 34.34it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:14<00:00,  7.31it/s]\n",
            "2023-05-19 11:58:57   < test - #q: 105; #db: 12771 >: R@1: 71.4, R@5: 81.9, R@10: 90.5, R@20: 93.3\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --multi_scale --multi_scale_method=sum --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2 --ensemble_merge_preds True"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "v446WUrZVzJN"
      },
      "source": [
        "### MAX"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-NguT9PjWWtM",
        "outputId": "8e1ca215-9673-4182-a8e4-e0dcde217c39"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 11:59:01   eval_ensemble.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True\n",
            "2023-05-19 11:59:01   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/small/test')\n",
            "2023-05-19 11:59:01   The outputs are being saved in logs/default/2023-05-19_11-59-01\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 11:59:02   Using efficient net v2s\n",
            "2023-05-19 11:59:02   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:59:03   Using efficient net v2s\n",
            "2023-05-19 11:59:03   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:59:03   Using efficient net v2s\n",
            "2023-05-19 11:59:03   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 11:59:06   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 11:59:06   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 11:59:06   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 11:59:06   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:25<00:00,  1.96it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:24<00:00, 40.46it/s]\n",
            "GRL: [1.0750091 1.105794  1.142057  1.1491336 1.1982573 1.2041144 1.2147048\n",
            " 1.240677  1.2434177 1.2997102 1.3016529 1.3246946 1.3656375 1.3656429\n",
            " 1.3739171 1.3777853 1.3822582 1.3836815 1.3842924 1.3892176], [   31    35    33    34    29    25    32    39    37    38    26 13523\n",
            "  2954   110 16121  1694  8827   321  4867  9255]\n",
            "2023-05-19 12:13:58   Test with multi-scale, the multi-scale method is: max\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [15:10<00:00,  1.87it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:25<00:00, 39.79it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:12<00:00,  7.52it/s]\n",
            "2023-05-19 12:31:46   < test - #q: 1000; #db: 27191 >: R@1: 66.4, R@5: 72.2, R@10: 76.4, R@20: 79.8\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v5bliKm_WWtN",
        "outputId": "469333fe-06b9-4895-f8c2-95a591499b49"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 12:31:50   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True\n",
            "2023-05-19 12:31:50   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-19 12:31:50   The outputs are being saved in logs/default/2023-05-19_12-31-50\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 12:31:51   Using efficient net v2s\n",
            "2023-05-19 12:31:51   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 12:31:52   Using efficient net v2s\n",
            "2023-05-19 12:31:52   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 12:31:53   Using efficient net v2s\n",
            "2023-05-19 12:31:53   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 12:31:55   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 12:31:55   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 12:31:55   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 12:31:56   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:47<00:00,  1.96it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:07<00:00, 42.45it/s]\n",
            "GRL: [1.0502496 1.113689  1.1741686 1.1842535 1.1848673 1.2011849 1.2465967\n",
            " 1.247319  1.2513933 1.256136  1.2566972 1.261983  1.2704458 1.2779176\n",
            " 1.2810608 1.2863259 1.2909162 1.2956309 1.2966197 1.3046079], [  685   703   627   713   714   699  1542  4726   740  7330 10390 12340\n",
            "  4722   553   463  6171   594  2153 11438  5091]\n",
            "2023-05-19 12:38:51   Test with multi-scale, the multi-scale method is: max\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:59<00:00,  1.90it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 38.75it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:40<00:00,  7.75it/s]\n",
            "2023-05-19 12:46:39   < test - #q: 315; #db: 12771 >: R@1: 85.4, R@5: 93.3, R@10: 95.2, R@20: 97.5\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AgidzUCtWWtO",
        "outputId": "eaca5b99-9165-4568-c426-7432ddbb3d3e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 12:46:43   eval_ensemble.py --dataset_folder /content/tokyo-night/ --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2 --ensemble_merge_preds True\n",
            "2023-05-19 12:46:43   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='max', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-19 12:46:43   The outputs are being saved in logs/default/2023-05-19_12-46-43\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 12:46:44   Using efficient net v2s\n",
            "2023-05-19 12:46:44   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 12:46:45   Using efficient net v2s\n",
            "2023-05-19 12:46:45   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 12:46:45   Using efficient net v2s\n",
            "2023-05-19 12:46:45   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 12:46:48   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 12:46:48   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 12:46:48   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 12:46:48   Test with multi-scale, the multi-scale method is: max\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:46<00:00,  1.97it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 34.80it/s]\n",
            "GRL: [1.3709009 1.4178642 1.4632802 1.4786114 1.481875  1.4861987 1.4900265\n",
            " 1.496777  1.4972763 1.4977115 1.4982603 1.5074619 1.5104132 1.5104392\n",
            " 1.5118623 1.5191473 1.5258915 1.5287527 1.5315249 1.5318456], [ 3571  1243 10513  5479  9052  3214 10492 11489  2121  3058   704  1096\n",
            "  3052 10522  8325  1652 11807  8675 10436 10520]\n",
            "2023-05-19 12:53:37   Test with multi-scale, the multi-scale method is: max\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:58<00:00,  1.91it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.70it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:13<00:00,  7.68it/s]\n",
            "2023-05-19 13:00:53   < test - #q: 105; #db: 12771 >: R@1: 75.2, R@5: 83.8, R@10: 91.4, R@20: 95.2\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --multi_scale --multi_scale_method=max --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2 --ensemble_merge_preds True"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "91HdiXh2V4Cl"
      },
      "source": [
        "### MIN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "19v1_tmqWao7",
        "outputId": "4019017f-0530-4220-d1cc-4c528d2f62d2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 13:00:57   eval_ensemble.py --dataset_folder /content/small/ --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True\n",
            "2023-05-19 13:00:57   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/small/test')\n",
            "2023-05-19 13:00:57   The outputs are being saved in logs/default/2023-05-19_13-00-57\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 13:00:57   Using efficient net v2s\n",
            "2023-05-19 13:00:57   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:00:58   Using efficient net v2s\n",
            "2023-05-19 13:00:58   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:00:59   Using efficient net v2s\n",
            "2023-05-19 13:00:59   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:01:01   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 13:01:01   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 13:01:01   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 13:01:02   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                      | 0/1700 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:16<00:00,  1.99it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:24<00:00, 41.23it/s]\n",
            "GRL: [1.1088043 1.1519785 1.1560986 1.1643776 1.2299229 1.2383301 1.2644205\n",
            " 1.2732866 1.2940571 1.3125322 1.3239781 1.3788006 1.3946166 1.3977673\n",
            " 1.398977  1.4025424 1.4100957 1.4245229 1.4297608 1.4307431], [   31    33    35    34    32    29    25    37    39    38    26 13523\n",
            "  8827  1694   110  2954   321  7465 14389  4867]\n",
            "2023-05-19 13:15:43   Test with multi-scale, the multi-scale method is: min\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [14:46<00:00,  1.92it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:24<00:00, 40.99it/s]\n",
            "Testing: 100%|██████████████████████████████████████████████████| 1000/1000 [02:09<00:00,  7.70it/s]\n",
            "2023-05-19 13:33:04   < test - #q: 1000; #db: 27191 >: R@1: 66.8, R@5: 72.4, R@10: 76.0, R@20: 79.6\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/small/  --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fT48_h32Wao8",
        "outputId": "a2d5a1c5-7a39-4ca1-cf02-d245994ddc1a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 13:33:07   eval_ensemble.py --dataset_folder /content/tokyo_xs/ --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True\n",
            "2023-05-19 13:33:07   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo_xs/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/tokyo_xs/test')\n",
            "2023-05-19 13:33:07   The outputs are being saved in logs/default/2023-05-19_13-33-07\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 13:33:08   Using efficient net v2s\n",
            "2023-05-19 13:33:08   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:33:09   Using efficient net v2s\n",
            "2023-05-19 13:33:09   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:33:10   Using efficient net v2s\n",
            "2023-05-19 13:33:10   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:33:12   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 13:33:12   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 13:33:12   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 13:33:12   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:44<00:00,  1.98it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:07<00:00, 43.31it/s]\n",
            "GRL: [1.0412743 1.128748  1.147159  1.1927478 1.2287343 1.2370157 1.2411427\n",
            " 1.2542989 1.2620982 1.2752272 1.2977158 1.2987831 1.3034234 1.3125951\n",
            " 1.3137337 1.3149589 1.3201314 1.3219953 1.3220661 1.3276297], [  685   703   714   713   627   699   740  7330 12340  1542  3310 12392\n",
            "  4722  2477  6171  4726   594  8820 10390   463]\n",
            "2023-05-19 13:40:04   Test with multi-scale, the multi-scale method is: min\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:55<00:00,  1.92it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:08<00:00, 38.59it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 315/315 [00:40<00:00,  7.85it/s]\n",
            "2023-05-19 13:47:48   < test - #q: 315; #db: 12771 >: R@1: 83.5, R@5: 90.8, R@10: 96.2, R@20: 97.5\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo_xs/  --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --ensemble_merge_preds True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NW7ezYkCWao9",
        "outputId": "e03b3759-0e8f-4157-d917-2cc2ce6abcb2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-05-19 13:47:51   eval_ensemble.py --dataset_folder /content/tokyo-night/ --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2 --ensemble_merge_preds True\n",
            "2023-05-19 13:47:51   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='efficientnet_v2_s', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model=None, device='cuda', seed=0, num_workers=8, dataset_folder='/content/tokyo-night/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=True, select_resolutions=[0.526, 0.588, 1.0, 1.7, 1.9], multi_scale_method='min', grl_param=0.3, night_test=True, night_brightness=0.2, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path='/content/eff2vs_grl/eff2vs_grl.pth', geowarp_model_path='/content/eff2vs_geowarp/best_model.pth', ensemble_merge_preds=True, test_set_folder='/content/tokyo-night/test')\n",
            "2023-05-19 13:47:51   The outputs are being saved in logs/default/2023-05-19_13-47-51\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=EfficientNet_V2_S_Weights.IMAGENET1K_V1`. You can also use `weights=EfficientNet_V2_S_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "2023-05-19 13:47:52   Using efficient net v2s\n",
            "2023-05-19 13:47:52   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:47:53   Using efficient net v2s\n",
            "2023-05-19 13:47:53   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:47:54   Using efficient net v2s\n",
            "2023-05-19 13:47:54   Model uses weights IMAGENET1K_V1\n",
            "2023-05-19 13:47:56   There are 1 GPUs and 12 CPUs.\n",
            "2023-05-19 13:47:56   Loading model from /content/eff2vs_grl/eff2vs_grl.pth\n",
            "2023-05-19 13:47:56   Loading model from /content/eff2vs_geowarp/best_model.pth\n",
            "2023-05-19 13:47:57   Test with multi-scale, the multi-scale method is: min\n",
            "  0%|                                                                       | 0/799 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).\n",
            "  warnings.warn(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [06:48<00:00,  1.96it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.64it/s]\n",
            "GRL: [1.4387205 1.4507562 1.4766145 1.4848017 1.49456   1.4956753 1.4980003\n",
            " 1.4981778 1.4982486 1.5100198 1.5171456 1.5217191 1.524045  1.5294828\n",
            " 1.5474124 1.549657  1.550682  1.5527847 1.5613412 1.5637742], [ 1243  3571 10513  9052 10492 10522 10436  3058  1096 11807 11489   879\n",
            "  3214 10520  3052 10441  3406   714  2953  3331]\n",
            "2023-05-19 13:54:48   Test with multi-scale, the multi-scale method is: min\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [07:03<00:00,  1.89it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:03<00:00, 34.63it/s]\n",
            "Testing: 100%|████████████████████████████████████████████████████| 105/105 [00:13<00:00,  7.73it/s]\n",
            "2023-05-19 14:02:08   < test - #q: 105; #db: 12771 >: R@1: 75.2, R@5: 84.8, R@10: 90.5, R@20: 95.2\n"
          ]
        }
      ],
      "source": [
        "!python eval_ensemble.py --dataset_folder /content/tokyo-night/  --multi_scale --multi_scale_method=min --select_resolution 0.526 0.588 1 1.7 1.9 --backbone efficientnet_v2_s --grl_param 0.3 --grl_model_path /content/eff2vs_grl/eff2vs_grl.pth --geowarp_model_path /content/eff2vs_geowarp/best_model.pth --night_test True --night_brightness 0.2 --ensemble_merge_preds True"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Qo_Rce7Jfd7_"
      },
      "source": [
        "-------------------------------------------------------------------------------------------------"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "D-WLgYLgdDOZ"
      },
      "source": [
        "- Tenere model soup per tre loss **(OK)** \n",
        "- cambiare nome file del model soup (evalEnsembler.py -> model_soups.py) **(OK)**\n",
        "- eliminare file primo ensemble (quello sulle tre loss)\n",
        "- controllare multi scale\n",
        "- verificare fattibilità test multi scale \n",
        "- CODE REVIEW\n",
        "- opzionale: geowarp+grl\n",
        "- fine.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VbzEO832jkkC",
        "outputId": "0e03c928-3e07-430e-f26b-979e335c955b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [03:45<00:00,  7.53it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:16<00:00, 61.36it/s]\n",
            "Potential greedy soup val acc 51.0, best so far 1.2.\n",
            "Adding to soup.\n",
            "100%|███████████████████████████████████████████████████████████| 1700/1700 [03:41<00:00,  7.69it/s]\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:16<00:00, 60.35it/s]\n",
            "Potential greedy soup val acc 52.0, best so far 51.0.\n",
            "Adding to soup.\n"
          ]
        }
      ],
      "source": [
        "!python /content/model_soup.py --dataset_folder /content/small"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zWe4r5RUY5ME",
        "outputId": "12d30bb7-e6c3-46d5-89f9-55fc9e5f266c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:05<00:00,  6.35it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:06<00:00, 51.05it/s]\n",
            "Potential greedy soup val acc 68.25396825396825, best so far 1.2.\n",
            "Adding to soup.\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:01<00:00,  6.55it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 315/315 [00:04<00:00, 65.36it/s]\n",
            "Potential greedy soup val acc 67.3015873015873, best so far 68.25396825396825.\n"
          ]
        }
      ],
      "source": [
        "!python /content/model_soup.py --dataset_folder /content/tokyo_xs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MNYtWrENbt-h",
        "outputId": "c7283703-e5c1-4136-caeb-1540eb9f3e37"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:07<00:00,  6.27it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 48.44it/s]\n",
            "Potential greedy soup val acc 45.714285714285715, best so far 1.2.\n",
            "Adding to soup.\n",
            "100%|█████████████████████████████████████████████████████████████| 799/799 [02:00<00:00,  6.63it/s]\n",
            "100%|█████████████████████████████████████████████████████████████| 105/105 [00:02<00:00, 35.28it/s]\n",
            "Potential greedy soup val acc 47.61904761904761, best so far 45.714285714285715.\n",
            "Adding to soup.\n"
          ]
        }
      ],
      "source": [
        "!python /content/model_soup.py --dataset_folder /content/tokyo-night"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "G8N9TjP4-JJu"
      },
      "source": [
        "# XAI (SHAP)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I1G2JQsL-NJY",
        "outputId": "40061447-b01f-4e0e-8616-14052989bbb1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2023-04-29 19:14:46   eval.py --dataset_folder /content/small/ --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "2023-04-29 19:14:46   Arguments: Namespace(wd=None, M=10, alpha=30, N=5, L=2, groups_num=8, min_images_per_class=10, backbone='ResNet18', fc_output_dim=512, loss_function='cosface', use_amp16=False, augmentation_device='cuda', batch_size=32, epochs_num=50, iterations_per_epoch=10000, lr=1e-05, classifiers_lr=0.01, brightness=0.7, contrast=0.7, hue=0.5, saturation=0.7, random_resized_crop=0.5, infer_batch_size=16, positive_dist_threshold=25, resume_train=None, resume_model='/content/logs/content/logs/default/trained_with_cosface/best_model.pth', device='cuda', seed=0, num_workers=8, dataset_folder='/content/small/', save_dir='default', k=0.6, ss_w=1, consistency_w=0.1, features_wise_w=10, qp_threshold=1.2, num_reranked_preds=5, kernel_sizes=[7, 5, 5, 5, 5, 5], channels=[225, 128, 128, 64, 64, 64, 64], multi_scale=False, select_resolutions=[0.526, 0.588, 1, 1.7, 1.9], multi_scale_method='avg', grl_param=None, night_test=False, night_brightness=0.1, source_dir=None, target_dir=None, test_method='hard_resize', optim='adam', resize=[480, 640], grl_model_path=None, geowarp_model_path=None, test_set_folder='/content/small/test')\n",
            "2023-04-29 19:14:46   The outputs are being saved in logs/default/2023-04-29_19-14-46\n",
            "2023-04-29 19:14:46   There are 1 GPUs and 2 CPUs.\n",
            "2023-04-29 19:14:46   Loading model from /content/logs/content/logs/default/trained_with_cosface/best_model.pth\n",
            "This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "2023-04-29 19:14:48   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:16<00:00, 60.39it/s]\n",
            "(1, 512)\n",
            "(27191, 512)\n",
            "(1000, 20)\n",
            "[[   34    31    32    36    35    38    33    29  1385    87    12    25\n",
            "  13022   100   199   456 23822  6884 12998   110]]\n",
            "[122  98 108  99 103 117 116 109 112 102 118 106 111 121  97  96 115 114\n",
            " 119 113 120 101  95 100 105  94 104  93 123]\n",
            "----------\n",
            "[  108    94    96    93    95    98 25056   109   105   106    99   102\n",
            "  9478    97  8017  7809  2223 24227 19996 13455]\n",
            "2023-04-29 19:15:05   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:05   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:05   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:06   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:06   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:06   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:07   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:08   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:08   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:09   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:10   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:11   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:12   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:12   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:13   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:14   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:14   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:15   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:16   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:17   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:18   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:19   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:20   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:21   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:23   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:25   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:26   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:27   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:29   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:30   Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
            "2023-04-29 19:15:31   < test - #q: 1000; #db: 27191 >: R@1: 0.1, R@5: 0.1, R@10: 0.1, R@20: 0.1\n"
          ]
        }
      ],
      "source": [
        "%matplotlib inline\n",
        "\n",
        "!python eval.py  --dataset_folder /content/small/  --resume_model /content/logs/content/logs/default/trained_with_cosface/best_model.pth #file in logs.zip"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "3_TcfYG2yfZo",
        "FQvJDmdJ1pj8",
        "TYeUeG4rwRiw"
      ],
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}