Step2a.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6dfcad30"
   },
   "source": [
    "# STEP 2: Refining representations with neural networks added to improved pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d139fea1"
   },
   "source": [
    "## Ranking function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 19,
     "status": "ok",
     "timestamp": 1649034476870,
     "user": {
      "displayName": "Neil Pillai",
      "userId": "06533696322730179220"
     },
     "user_tz": 240
    },
    "id": "12c7d566"
   },
   "outputs": [],
   "source": [
    "\"\"\"Retrieval ranking function for the learnt representations from the official code of im2recipe paper\"\"\"\n",
    "def ranker(im_vecs, instr_vecs, N = 1000, flag = \"image\"):\n",
    "    idxs = range(N)\n",
    "\n",
    "    glob_rank = []\n",
    "    glob_recall = {1:0.0,5:0.0,10:0.0}\n",
    "    for i in range(10):\n",
    "\n",
    "        ids = random.sample(range(0,len(im_vecs)), N)\n",
    "        \n",
    "        im_sub = im_vecs[ids,:]\n",
    "        instr_sub = instr_vecs[ids,:]\n",
    "\n",
    "        if flag == \"image\":\n",
    "            sims = np.dot(im_sub,instr_sub.T) # for im2recipe\n",
    "        else:\n",
    "            sims = np.dot(instr_sub,im_sub.T) # for recipe2im\n",
    "\n",
    "        med_rank = []\n",
    "        recall = {1:0.0,5:0.0,10:0.0}\n",
    "\n",
    "        for ii in idxs:\n",
    "\n",
    "            # name = ids_sub[ii]\n",
    "            # get a column of similarities\n",
    "            sim = sims[ii,:]\n",
    "\n",
    "            # sort indices in descending order\n",
    "            sorting = np.argsort(sim)[::-1].tolist()\n",
    "\n",
    "            # find where the index of the pair sample ended up in the sorting\n",
    "            pos = sorting.index(ii)\n",
    "\n",
    "            if (pos+1) == 1:\n",
    "                recall[1]+=1\n",
    "            if (pos+1) <=5:\n",
    "                recall[5]+=1\n",
    "            if (pos+1)<=10:\n",
    "                recall[10]+=1\n",
    "\n",
    "            # store the position\n",
    "            med_rank.append(pos+1)\n",
    "\n",
    "        for i in recall.keys():\n",
    "            recall[i]=recall[i]/N\n",
    "\n",
    "        med = np.median(med_rank)\n",
    "\n",
    "        for i in recall.keys():\n",
    "            glob_recall[i]+=recall[i]\n",
    "        glob_rank.append(med)\n",
    "\n",
    "    for i in glob_recall.keys():\n",
    "        glob_recall[i] = glob_recall[i]/10\n",
    "    \n",
    "    print (\"Mean median\", np.average(glob_rank))\n",
    "    print (\"Recall\", glob_recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4bf6eb3a-48b0-457a-af11-86238c12c079"
   },
   "source": [
    "# Loading necessary files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 116524,
     "status": "ok",
     "timestamp": 1649034476869,
     "user": {
      "displayName": "Neil Pillai",
      "userId": "06533696322730179220"
     },
     "user_tz": 240
    },
    "id": "64504e16-fae1-40de-9dc2-ed26ef94f551"
   },
   "outputs": [],
   "source": [
    "# validation image and full text data\n",
    "img_val = torch.load(\"img_val.pt\")\n",
    "text_val = torch.load(\"text_val.pt\")\n",
    "\n",
    "# train image and full text data\n",
    "img_train = torch.load(\"img_train.pt\")\n",
    "text_train = torch.load(\"text_train.pt\")\n",
    "\n",
    "# test image and full text data\n",
    "img_test = torch.load(\"img_test.pt\")\n",
    "text_test = torch.load(\"text_test.pt\")\n",
    "\n",
    "#individual text test data\n",
    "ingredients_test = torch.load(\"test_ingredients.pt\")\n",
    "instructions_test = torch.load(\"test_instructions.pt\")\n",
    "title_test = torch.load(\"test_title.pt\")\n",
    "\n",
    "#individual text train data\n",
    "ingredients_train = torch.load(\"train_ingredients.pt\")\n",
    "instructions_train = torch.load(\"train_instructions.pt\")\n",
    "title_train = torch.load(\"train_title.pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "df7f444d-cfff-43ec-97dc-c92d837a69d2"
   },
   "source": [
    "## 2a: Using MSE Loss to train NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "d95bb898-c671-41e0-8907-5616e6907542"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms, utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "ef6fa414-9e21-4783-9299-dc112db34c4c"
   },
   "outputs": [],
   "source": [
    "!export CUDA_VISIBLE_DEVICES='1,2,3,4'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "db9def68-099e-486a-82b0-048edf70fd73"
   },
   "source": [
    "device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3c05f885"
   },
   "source": [
    "### Model definition and train loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "4c1c7ac0-20ee-4838-bff7-d70cdc11946e"
   },
   "outputs": [],
   "source": [
    "class EmbeddingDataset(Dataset):\n",
    "    def __init__(self, image_emb, text_emb, transform=None):\n",
    "        self.image_emb = torch.as_tensor(np.array(image_emb))\n",
    "        self.text_emb = torch.as_tensor(np.array(text_emb))        \n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.image_emb)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.image_emb[idx], self.text_emb[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "f656948f-ac43-4e9d-aba7-fee81028079b"
   },
   "outputs": [],
   "source": [
    "class EmbeddingNetwork(nn.Module):\n",
    "    def __init__(self, output_size, input_size=1024):\n",
    "        super().__init__()\n",
    "\n",
    "        self.layer1 = nn.Sequential(\n",
    "            nn.Linear(input_size, 512),\n",
    "            nn.BatchNorm1d(512),\n",
    "            nn.Dropout(),\n",
    "            nn.LeakyReLU()\n",
    "        )\n",
    "        self.layer2 = nn.Linear(512, output_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layer1(x)\n",
    "        return self.layer2(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AverageMeter(object):\n",
    "    # Utility function for timers\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "    \n",
    "    def reset(self):\n",
    "        self.val, self.avg, self.sum, self.count = 0, 0, 0, 0\n",
    "    \n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(train_loader, img_model, txt_model, criterion, optimizer, epoch):\n",
    "    print('Starting training epoch {}'.format(epoch))\n",
    "    img_model.train()\n",
    "    txt_model.train()\n",
    "    \n",
    "    batch_time, data_time, losses = AverageMeter(), AverageMeter(), AverageMeter()\n",
    "    end = time.time()\n",
    "    running_loss = 0.\n",
    "    last_loss = 0.\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    for i, (image_emb, text_emb) in enumerate(train_loader):\n",
    "    \n",
    "        # Use GPU if available\n",
    "        if use_gpu: \n",
    "            image_emb, text_emb = image_emb.to(f'cuda:{img_model.device_ids[0]}'), text_emb.to(f'cuda:{txt_model.device_ids[0]}')\n",
    "\n",
    "        data_time.update(time.time() - end)\n",
    "\n",
    "        # Run forward pass\n",
    "        out_image_emb = img_model(image_emb) \n",
    "        out_text_emb = txt_model(text_emb)\n",
    "        loss = criterion(out_image_emb, out_text_emb) \n",
    "\n",
    "        # Compute gradient and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        batch_time.update(time.time() - end)\n",
    "        end = time.time()\n",
    "\n",
    "        # Print model accuracy -- in the code below\n",
    "        running_loss += loss.item()\n",
    "        if i % 2000 == 0:\n",
    "            last_loss = running_loss / 2000 # loss per batch\n",
    "            print('  batch {} loss: {}'.format(i + 1, last_loss))\n",
    "            running_loss = 0.\n",
    "        \n",
    "        if i % 2000 == 0:\n",
    "            print('Epoch: [{0}][{1}/{2}]\\t'\n",
    "                'Time {batch_time.val} ({batch_time.avg})\\t'\n",
    "                'Data {data_time.val} ({data_time.avg})\\t'.format(\n",
    "                  epoch, i, len(train_loader), batch_time=batch_time,\n",
    "                 data_time=data_time)) \n",
    "\n",
    "    print('Finished training epoch {}'.format(epoch))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training: Dim = 512; all components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "75afd4aa"
   },
   "source": [
    "#### im2recipe and recipe2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fd7a7bb5-5a25-4b22-8320-f5592174989b"
   },
   "outputs": [],
   "source": [
    "img_model = EmbeddingNetwork(512)\n",
    "img_model= nn.DataParallel(img_model, device_ids=[1,2,3])\n",
    "img_model.to(device);\n",
    "\n",
    "txt_model = EmbeddingNetwork(512);\n",
    "txt_model= nn.DataParallel(txt_model, device_ids=[1,2,3])\n",
    "txt_model.to(device);\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to(device)\n",
    "    img_model = img_model.to(device)\n",
    "    txt_model = txt_model.to(device)\n",
    "    \n",
    "train_dataset = EmbeddingDataset(img_train, text_train)\n",
    "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "best_losses = 1e10\n",
    "epochs = 10\n",
    "for epoch in range(epochs):\n",
    "\n",
    "    train(train_loader, img_model, txt_model, criterion, optimizer, epoch)\n",
    "        \n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "torch.save(img_model.state_dict(), 'checkpoints/img-model-full-512-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model.state_dict(), 'checkpoints/txt-model-full-512-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f9add441"
   },
   "source": [
    "#### im2title and title2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fcffa099"
   },
   "outputs": [],
   "source": [
    "title_dataset = EmbeddingDataset(img_train, title_train)\n",
    "title_loader = DataLoader(title_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "img_model_title = EmbeddingNetwork(512)\n",
    "img_model_title= nn.DataParallel(img_model_title, device_ids=[1,2,3])\n",
    "img_model_title.to(device);\n",
    "\n",
    "txt_model_title = EmbeddingNetwork(512);\n",
    "txt_model_title= nn.DataParallel(txt_model_title, device_ids=[1,2,3])\n",
    "txt_model_title.to(device);\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_title.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    img_model_title = txt_model_title.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    txt_model_title = txt_model_title.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    \n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "\n",
    "    train(title_loader, img_model_title, txt_model_title, criterion, optimizer, epoch)\n",
    "        \n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "    torch.save(img_model_title.state_dict(), 'checkpoints/img-model-title-512-epoch-{}.pth'.format(epoch+1))\n",
    "    torch.save(txt_model_title.state_dict(), 'checkpoints/txt-model-title-512-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9663baa6"
   },
   "source": [
    "#### im2ingredients and ingredients2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0a9f244c",
    "outputId": "3b71a0b6-df8c-42f9-fd5f-8db94ed8f016"
   },
   "outputs": [],
   "source": [
    "img_model_ingredients = EmbeddingNetwork(512)\n",
    "img_model_ingredients= nn.DataParallel(img_model_ingredients, device_ids=[1,2,3])\n",
    "img_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'));\n",
    "\n",
    "txt_model_ingredients = EmbeddingNetwork(512)\n",
    "txt_model_ingredients= nn.DataParallel(txt_model_ingredients, device_ids=[1,2,3])\n",
    "txt_model_ingredients.to((f'cuda:{txt_model_ingredients.device_ids[0]}'));\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_ingredients.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "ingredients_dataset = EmbeddingDataset(img_train, ingredients_train)\n",
    "ingredients_loader = DataLoader(ingredients_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "    img_model_ingredients = img_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "    txt_model_ingredients = txt_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(ingredients_loader, img_model_ingredients, txt_model_ingredients, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "\n",
    "torch.save(img_model_ingredients.state_dict(), 'checkpoints/img-model-ingredients-512-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_ingredients.state_dict(), 'checkpoints/txt-model-ingredients-512-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3062b9f9"
   },
   "source": [
    "#### im2instructions and instructions2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "de2689a3",
    "outputId": "4923b29d-9cb8-4954-f192-deea8a109276"
   },
   "outputs": [],
   "source": [
    "img_model_instructions = EmbeddingNetwork(512)\n",
    "img_model_instructions= nn.DataParallel(img_model_instructions, device_ids=[1,2,3])\n",
    "img_model_instructions.to((f'cuda:{img_model_instructions.device_ids[0]}'));\n",
    "txt_model_instructions = EmbeddingNetwork(512)\n",
    "txt_model_instructions= nn.DataParallel(txt_model_instructions, device_ids=[1,2,3])\n",
    "txt_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'));\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_instructions.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "instructions_dataset = EmbeddingDataset(img_train, instructions_train)\n",
    "instructions_loader = DataLoader(instructions_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "    img_model_instructions = img_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "    txt_model_instructions = txt_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(ingredients_loader, img_model_instructions, txt_model_instructions, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "\n",
    "torch.save(img_model_instructions.state_dict(), 'checkpoints/img-model-instructions-512-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_instructions.state_dict(), 'checkpoints/txt-model-instructions-512-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "364fd19b"
   },
   "source": [
    "### Training: dims = 256; all components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "37c7aeb7"
   },
   "source": [
    "#### im2recipe and recipe2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gscCkfG1LWWN",
    "outputId": "7ed3a2b0-c885-4100-dd0b-4ad8f4f47242"
   },
   "outputs": [],
   "source": [
    "img_model = EmbeddingNetwork(256)\n",
    "img_model= nn.DataParallel(img_model, device_ids=[2,3])\n",
    "img_model.to(device);\n",
    "txt_model = EmbeddingNetwork(256);\n",
    "txt_model= nn.DataParallel(txt_model, device_ids=[2,3])\n",
    "txt_model.to(device);\n",
    "optimizer = torch.optim.Adam(img_model.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to(device)\n",
    "    img_model = img_model.to(device)\n",
    "    txt_model = txt_model.to(device)\n",
    "\n",
    "train_dataset = EmbeddingDataset(img_train, text_train)\n",
    "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "best_losses = 1e10\n",
    "epochs = 10\n",
    "for epoch in range(epochs):\n",
    "    train(train_loader, img_model, txt_model, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "torch.save(img_model.state_dict(), 'checkpoints/img-model-full-256-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model.state_dict(), 'checkpoints/txt-model-full-256-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lTmZH8KqLWWN"
   },
   "source": [
    "#### im2title and title2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GbEwGQz1LWWO",
    "outputId": "35578813-9148-45a2-b398-7da66dc6eb07"
   },
   "outputs": [],
   "source": [
    "title_dataset = EmbeddingDataset(img_train, title_train)\n",
    "title_loader = DataLoader(title_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "img_model_title = EmbeddingNetwork(256)\n",
    "img_model_title= nn.DataParallel(img_model_title, device_ids=[2,3])\n",
    "img_model_title.to(device);\n",
    "\n",
    "txt_model_title = EmbeddingNetwork(256);\n",
    "txt_model_title= nn.DataParallel(txt_model_title, device_ids=[2,3])\n",
    "txt_model_title.to(device);\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    img_model_title = txt_model_title.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    txt_model_title = txt_model_title.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(title_loader, img_model_title, txt_model_title, criterion, optimizer, epoch)\n",
    "        \n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "torch.save(img_model_title.state_dict(), 'checkpoints/img-model-title-256-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_title.state_dict(), 'checkpoints/txt-model-title-256-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6Rr1WETBLWWO"
   },
   "source": [
    "#### im2ingredients and ingredients2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "309dac99",
    "outputId": "3ff40ef0-e0e3-43ff-f3d5-cba64ea6645b"
   },
   "outputs": [],
   "source": [
    "img_model_ingredients = EmbeddingNetwork(256)\n",
    "img_model_ingredients= nn.DataParallel(img_model_ingredients, device_ids=[2,3])\n",
    "img_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'));\n",
    "\n",
    "txt_model_ingredients = EmbeddingNetwork(256)\n",
    "txt_model_ingredients= nn.DataParallel(txt_model_ingredients, device_ids=[2,3])\n",
    "txt_model_ingredients.to((f'cuda:{txt_model_ingredients.device_ids[0]}'));\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_ingredients.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "ingredients_dataset = EmbeddingDataset(img_train, ingredients_train)\n",
    "ingredients_loader = DataLoader(ingredients_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "    img_model_ingredients = img_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "    txt_model_ingredients = txt_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(ingredients_loader, img_model_ingredients, txt_model_ingredients, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "\n",
    "torch.save(img_model_ingredients.state_dict(), 'checkpoints/img-model-ingredients-256-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_ingredients.state_dict(), 'checkpoints/txt-model-ingredients-256-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GFMxYQRpLWWO"
   },
   "source": [
    "#### im2instructions and instructions2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4069577e",
    "outputId": "44dd2405-0975-4689-d943-546bd4028134"
   },
   "outputs": [],
   "source": [
    "img_model_instructions = EmbeddingNetwork(256)\n",
    "img_model_instructions= nn.DataParallel(img_model_instructions, device_ids=[2,3])\n",
    "img_model_instructions.to((f'cuda:{img_model_instructions.device_ids[0]}'));\n",
    "\n",
    "txt_model_instructions = EmbeddingNetwork(256)\n",
    "txt_model_instructions= nn.DataParallel(txt_model_instructions, device_ids=[2,3])\n",
    "txt_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'));\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_instructions.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "instructions_dataset = EmbeddingDataset(img_train, instructions_train)\n",
    "instructions_loader = DataLoader(instructions_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "    img_model_instructions = img_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "    txt_model_instructions = txt_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(ingredients_loader, img_model_instructions, txt_model_instructions, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "\n",
    "torch.save(img_model_instructions.state_dict(), 'checkpoints/img-model-instructions-256-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_instructions.state_dict(), 'checkpoints/txt-model-instructions-256-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training: dims = 128; all components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "37c7aeb7"
   },
   "source": [
    "#### im2recipe and recipe2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gscCkfG1LWWN",
    "outputId": "7ed3a2b0-c885-4100-dd0b-4ad8f4f47242"
   },
   "outputs": [],
   "source": [
    "img_model = EmbeddingNetwork(128)\n",
    "img_model= nn.DataParallel(img_model, device_ids=[2,3])\n",
    "img_model.to(device);\n",
    "txt_model = EmbeddingNetwork(128);\n",
    "txt_model= nn.DataParallel(txt_model, device_ids=[2,3])\n",
    "txt_model.to(device);\n",
    "optimizer = torch.optim.Adam(img_model.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to(device)\n",
    "    img_model = img_model.to(device)\n",
    "    txt_model = txt_model.to(device)\n",
    "\n",
    "train_dataset = EmbeddingDataset(img_train, text_train)\n",
    "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "best_losses = 1e10\n",
    "epochs = 10\n",
    "for epoch in range(epochs):\n",
    "    train(train_loader, img_model, txt_model, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "torch.save(img_model.state_dict(), 'checkpoints/img-model-full-128-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model.state_dict(), 'checkpoints/txt-model-full-128-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lTmZH8KqLWWN"
   },
   "source": [
    "#### im2title and title2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GbEwGQz1LWWO",
    "outputId": "35578813-9148-45a2-b398-7da66dc6eb07"
   },
   "outputs": [],
   "source": [
    "title_dataset = EmbeddingDataset(img_train, title_train)\n",
    "title_loader = DataLoader(title_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "img_model_title = EmbeddingNetwork(128)\n",
    "img_model_title= nn.DataParallel(img_model_title, device_ids=[2,3])\n",
    "img_model_title.to(device);\n",
    "\n",
    "txt_model_title = EmbeddingNetwork(128);\n",
    "txt_model_title= nn.DataParallel(txt_model_title, device_ids=[2,3])\n",
    "txt_model_title.to(device);\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    img_model_title = txt_model_title.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    txt_model_title = txt_model_title.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(title_loader, img_model_title, txt_model_title, criterion, optimizer, epoch)\n",
    "        \n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "torch.save(img_model_title.state_dict(), 'checkpoints/img-model-title-128-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_title.state_dict(), 'checkpoints/txt-model-title-128-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6Rr1WETBLWWO"
   },
   "source": [
    "#### im2ingredients and ingredients2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "309dac99",
    "outputId": "3ff40ef0-e0e3-43ff-f3d5-cba64ea6645b"
   },
   "outputs": [],
   "source": [
    "img_model_ingredients = EmbeddingNetwork(128)\n",
    "img_model_ingredients= nn.DataParallel(img_model_ingredients, device_ids=[2,3])\n",
    "img_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'));\n",
    "\n",
    "txt_model_ingredients = EmbeddingNetwork(128)\n",
    "txt_model_ingredients= nn.DataParallel(txt_model_ingredients, device_ids=[2,3])\n",
    "txt_model_ingredients.to((f'cuda:{txt_model_ingredients.device_ids[0]}'));\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_ingredients.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "ingredients_dataset = EmbeddingDataset(img_train, ingredients_train)\n",
    "ingredients_loader = DataLoader(ingredients_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "    img_model_ingredients = img_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "    txt_model_ingredients = txt_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(ingredients_loader, img_model_ingredients, txt_model_ingredients, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "\n",
    "torch.save(img_model_ingredients.state_dict(), 'checkpoints/img-model-ingredients-128-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_ingredients.state_dict(), 'checkpoints/txt-model-ingredients-128-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GFMxYQRpLWWO"
   },
   "source": [
    "#### im2instructions and instructions2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4069577e",
    "outputId": "44dd2405-0975-4689-d943-546bd4028134"
   },
   "outputs": [],
   "source": [
    "img_model_instructions = EmbeddingNetwork(128)\n",
    "img_model_instructions= nn.DataParallel(img_model_instructions, device_ids=[2,3])\n",
    "img_model_instructions.to((f'cuda:{img_model_instructions.device_ids[0]}'));\n",
    "\n",
    "txt_model_instructions = EmbeddingNetwork(128)\n",
    "txt_model_instructions= nn.DataParallel(txt_model_instructions, device_ids=[2,3])\n",
    "txt_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'));\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_instructions.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "instructions_dataset = EmbeddingDataset(img_train, instructions_train)\n",
    "instructions_loader = DataLoader(instructions_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "    img_model_instructions = img_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "    txt_model_instructions = txt_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(ingredients_loader, img_model_instructions, txt_model_instructions, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "\n",
    "torch.save(img_model_instructions.state_dict(), 'checkpoints/img-model-instructions-128-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_instructions.state_dict(), 'checkpoints/txt-model-instructions-128-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training: dims = 64; all components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "37c7aeb7"
   },
   "source": [
    "#### im2recipe and recipe2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gscCkfG1LWWN",
    "outputId": "7ed3a2b0-c885-4100-dd0b-4ad8f4f47242"
   },
   "outputs": [],
   "source": [
    "img_model = EmbeddingNetwork(64)\n",
    "img_model= nn.DataParallel(img_model, device_ids=[2,3])\n",
    "img_model.to(device);\n",
    "txt_model = EmbeddingNetwork(64);\n",
    "txt_model= nn.DataParallel(txt_model, device_ids=[2,3])\n",
    "txt_model.to(device);\n",
    "optimizer = torch.optim.Adam(img_model.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to(device)\n",
    "    img_model = img_model.to(device)\n",
    "    txt_model = txt_model.to(device)\n",
    "\n",
    "train_dataset = EmbeddingDataset(img_train, text_train)\n",
    "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "best_losses = 1e10\n",
    "epochs = 10\n",
    "for epoch in range(epochs):\n",
    "    train(train_loader, img_model, txt_model, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "torch.save(img_model.state_dict(), 'checkpoints/img-model-full-64-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model.state_dict(), 'checkpoints/txt-model-full-64-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lTmZH8KqLWWN"
   },
   "source": [
    "#### im2title and title2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GbEwGQz1LWWO",
    "outputId": "35578813-9148-45a2-b398-7da66dc6eb07"
   },
   "outputs": [],
   "source": [
    "title_dataset = EmbeddingDataset(img_train, title_train)\n",
    "title_loader = DataLoader(title_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "img_model_title = EmbeddingNetwork(64)\n",
    "img_model_title= nn.DataParallel(img_model_title, device_ids=[2,3])\n",
    "img_model_title.to(device);\n",
    "\n",
    "txt_model_title = EmbeddingNetwork(64);\n",
    "txt_model_title= nn.DataParallel(txt_model_title, device_ids=[2,3])\n",
    "txt_model_title.to(device);\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    img_model_title = txt_model_title.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "    txt_model_title = txt_model_title.to((f'cuda:{txt_model_title.device_ids[0]}'))\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(title_loader, img_model_title, txt_model_title, criterion, optimizer, epoch)\n",
    "        \n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "torch.save(img_model_title.state_dict(), 'checkpoints/img-model-title-64-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_title.state_dict(), 'checkpoints/txt-model-title-64-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6Rr1WETBLWWO"
   },
   "source": [
    "#### im2ingredients and ingredients2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "309dac99",
    "outputId": "3ff40ef0-e0e3-43ff-f3d5-cba64ea6645b"
   },
   "outputs": [],
   "source": [
    "img_model_ingredients = EmbeddingNetwork(64)\n",
    "img_model_ingredients= nn.DataParallel(img_model_ingredients, device_ids=[2,3])\n",
    "img_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'));\n",
    "\n",
    "txt_model_ingredients = EmbeddingNetwork(64)\n",
    "txt_model_ingredients= nn.DataParallel(txt_model_ingredients, device_ids=[2,3])\n",
    "txt_model_ingredients.to((f'cuda:{txt_model_ingredients.device_ids[0]}'));\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_ingredients.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "ingredients_dataset = EmbeddingDataset(img_train, ingredients_train)\n",
    "ingredients_loader = DataLoader(ingredients_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "    img_model_ingredients = img_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "    txt_model_ingredients = txt_model_ingredients.to((f'cuda:{img_model_ingredients.device_ids[0]}'))\n",
    "\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(ingredients_loader, img_model_ingredients, txt_model_ingredients, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "\n",
    "torch.save(img_model_ingredients.state_dict(), 'checkpoints/img-model-ingredients-64-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_ingredients.state_dict(), 'checkpoints/txt-model-ingredients-64-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GFMxYQRpLWWO"
   },
   "source": [
    "#### im2instructions and instructions2im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4069577e",
    "outputId": "44dd2405-0975-4689-d943-546bd4028134"
   },
   "outputs": [],
   "source": [
    "img_model_instructions = EmbeddingNetwork(64)\n",
    "img_model_instructions= nn.DataParallel(img_model_instructions, device_ids=[2,3])\n",
    "img_model_instructions.to((f'cuda:{img_model_instructions.device_ids[0]}'));\n",
    "\n",
    "txt_model_instructions = EmbeddingNetwork(64)\n",
    "txt_model_instructions= nn.DataParallel(txt_model_instructions, device_ids=[2,3])\n",
    "txt_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'));\n",
    "\n",
    "optimizer = torch.optim.Adam(img_model_instructions.parameters(), lr=1e-2, weight_decay=0.0)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "instructions_dataset = EmbeddingDataset(img_train, instructions_train)\n",
    "instructions_loader = DataLoader(instructions_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False\n",
    "if use_gpu: \n",
    "    criterion = criterion.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "    img_model_instructions = img_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "    txt_model_instructions = txt_model_instructions.to((f'cuda:{txt_model_instructions.device_ids[0]}'))\n",
    "\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    train(ingredients_loader, img_model_instructions, txt_model_instructions, criterion, optimizer, epoch)\n",
    "  # Save checkpoint and replace old best model if current model is betterabs\n",
    "\n",
    "torch.save(img_model_instructions.state_dict(), 'checkpoints/img-model-instructions-64-epoch-{}.pth'.format(epoch+1))\n",
    "torch.save(txt_model_instructions.state_dict(), 'checkpoints/txt-model-instructions-64-epoch-{}.pth'.format(epoch+1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ad96b882"
   },
   "source": [
    "### Dimensional Analysis on val data. Dims: [64, 128, 256, 512]; all components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "296c17c0"
   },
   "source": [
    "#### 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "fa8a4ffb"
   },
   "outputs": [],
   "source": [
    "# im2recipe 512\n",
    "img_model_full_512 = EmbeddingNetwork(512)\n",
    "img_model_full_512 = nn.DataParallel(img_model_full_512, device_ids=[1])\n",
    "img_model_full_512.load_state_dict(torch.load(\"checkpoints/img-model-full-512-epoch-10.pth\"))\n",
    "img_model_full_512.to((f'cuda:{img_model_full_512.device_ids[0]}'));\n",
    "img_model_full_512.eval();\n",
    "txt_model_full_512 = EmbeddingNetwork(512)\n",
    "txt_model_full_512 = nn.DataParallel(txt_model_full_512, device_ids=[1])\n",
    "txt_model_full_512.load_state_dict(torch.load(\"checkpoints/txt-model-full-512-epoch-10.pth\"))\n",
    "txt_model_full_512.to((f'cuda:{txt_model_full_512.device_ids[0]}'));\n",
    "txt_model_full_512.eval();\n",
    "\n",
    "#im2title 512\n",
    "img_model_title_512 = EmbeddingNetwork(512)\n",
    "img_model_title_512 = nn.DataParallel(img_model_title_512, device_ids=[1])\n",
    "img_model_title_512.load_state_dict(torch.load(\"checkpoints/img-model-title-512-epoch-5.pth\"))\n",
    "img_model_title_512.to((f'cuda:{img_model_title_512.device_ids[0]}'));\n",
    "img_model_title_512.eval();\n",
    "txt_model_title_512 = EmbeddingNetwork(512)\n",
    "txt_model_title_512 = nn.DataParallel(txt_model_title_512, device_ids=[1])\n",
    "txt_model_title_512.load_state_dict(torch.load(\"checkpoints/txt-model-title-512-epoch-5.pth\"))\n",
    "txt_model_title_512.to((f'cuda:{txt_model_title_512.device_ids[0]}'));\n",
    "txt_model_title_512.eval();\n",
    "\n",
    "#im2instructions 512\n",
    "img_model_instructions_512 = EmbeddingNetwork(512)\n",
    "img_model_instructions_512 = nn.DataParallel(img_model_instructions_512, device_ids=[1])\n",
    "img_model_instructions_512.load_state_dict(torch.load(\"checkpoints/img-model-instructions-512-epoch-5.pth\"))\n",
    "img_model_instructions_512.to((f'cuda:{img_model_instructions_512.device_ids[0]}'));\n",
    "img_model_instructions_512.eval();\n",
    "txt_model_instructions_512 = EmbeddingNetwork(512)\n",
    "txt_model_instructions_512 = nn.DataParallel(txt_model_instructions_512, device_ids=[1])\n",
    "txt_model_instructions_512.load_state_dict(torch.load(\"checkpoints/txt-model-instructions-512-epoch-5.pth\"))\n",
    "txt_model_instructions_512.to((f'cuda:{txt_model_instructions_512.device_ids[0]}'));\n",
    "txt_model_instructions_512.eval();\n",
    "\n",
    "#im2ingredients 512\n",
    "img_model_ingredients_512 = EmbeddingNetwork(512)\n",
    "img_model_ingredients_512 = nn.DataParallel(img_model_ingredients_512, device_ids=[1])\n",
    "img_model_ingredients_512.load_state_dict(torch.load(\"checkpoints/img-model-ingredients-512-epoch-5.pth\"))\n",
    "img_model_ingredients_512.to((f'cuda:{img_model_full_512.device_ids[0]}'));\n",
    "img_model_ingredients_512.eval();\n",
    "txt_model_ingredients_512 = EmbeddingNetwork(512)\n",
    "txt_model_ingredients_512 = nn.DataParallel(txt_model_ingredients_512, device_ids=[1])\n",
    "txt_model_ingredients_512.load_state_dict(torch.load(\"checkpoints/txt-model-ingredients-512-epoch-5.pth\"))\n",
    "txt_model_ingredients_512.to((f'cuda:{txt_model_ingredients_512.device_ids[0]}'));\n",
    "txt_model_ingredients_512.eval();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Full recipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "4cf32bae",
    "outputId": "cea10112-e0c6-48e3-d07c-b8e20e61947d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2recipe for dims = 512 and sample = 1000\n",
      "Mean median 4.9\n",
      "Recall {1: 0.24130000000000001, 5: 0.5218, 10: 0.6375}\n",
      "Running im2recipe for dims = 512 and sample = 10000\n",
      "Mean median 41.5\n",
      "Recall {1: 0.060719999999999996, 5: 0.19102, 10: 0.27913}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 512))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 512))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_full_512(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_full_512(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2recipe and recipe2im\n",
    "print(\"Running im2recipe for dims = 512 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2recipe for dims = 512 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "20628288",
    "outputId": "718fdc4c-9dee-42de-b18a-bf51f33a2b8f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2title for dims = 512 and sample = 1000\n",
      "Mean median 1.45\n",
      "Recall {1: 0.49539999999999995, 5: 0.7586, 10: 0.8249000000000001}\n",
      "Running im2title for dims = 512 and sample = 10000\n",
      "Mean median 7.0\n",
      "Recall {1: 0.2185, 5: 0.4593799999999999, 10: 0.56763}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 512))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 512))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_title_512(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_title_512(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2title and title2im\n",
    "print(\"Running im2title for dims = 512 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2title for dims = 512 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingredients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "13f66231",
    "outputId": "02314dba-10af-4109-9845-673a9cdab1ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2ingredients for dims = 512 and sample = 1000\n",
      "Mean median 11.9\n",
      "Recall {1: 0.1045, 5: 0.3227999999999999, 10: 0.4681}\n",
      "Running im2ingredients for dims = 512 and sample = 10000\n",
      "Mean median 112.0\n",
      "Recall {1: 0.01713, 5: 0.06742000000000001, 10: 0.11599}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 512))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 512))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_ingredients_512(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_ingredients_512(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2instructions and instructions2im\n",
    "print(\"Running im2ingredients for dims = 512 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2ingredients for dims = 512 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instructions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "5ac0916c",
    "outputId": "5d5d9cf4-9708-4921-fd53-8388d275b7ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2instructions for dims = 512 and sample = 1000\n",
      "Mean median 9.15\n",
      "Recall {1: 0.13710000000000003, 5: 0.38470000000000004, 10: 0.5317000000000001}\n",
      "Running im2instructions for dims = 512 and sample = 10000\n",
      "Mean median 84.15\n",
      "Recall {1: 0.0245, 5: 0.09233, 10: 0.15209}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 512))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 512))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_instructions_512(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_instructions_512(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2ingredients and ingredients2im\n",
    "print(\"Running im2instructions for dims = 512 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2instructions for dims = 512 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5c60c319"
   },
   "source": [
    "#### 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "f8fddd89"
   },
   "outputs": [],
   "source": [
    "# im2recipe 256\n",
    "img_model_full_256 = EmbeddingNetwork(256)\n",
    "img_model_full_256 = nn.DataParallel(img_model_full_256, device_ids=[1])\n",
    "img_model_full_256.load_state_dict(torch.load(\"checkpoints/img-model-full-256-epoch-10.pth\"))\n",
    "img_model_full_256.to((f'cuda:{img_model_full_256.device_ids[0]}'));\n",
    "img_model_full_256.eval();\n",
    "txt_model_full_256 = EmbeddingNetwork(256)\n",
    "txt_model_full_256 = nn.DataParallel(txt_model_full_256, device_ids=[1])\n",
    "txt_model_full_256.load_state_dict(torch.load(\"checkpoints/txt-model-full-256-epoch-10.pth\"))\n",
    "txt_model_full_256.to((f'cuda:{txt_model_full_256.device_ids[0]}'));\n",
    "txt_model_full_256.eval();\n",
    "\n",
    "#im2title 256\n",
    "img_model_title_256 = EmbeddingNetwork(256)\n",
    "img_model_title_256 = nn.DataParallel(img_model_title_256, device_ids=[1])\n",
    "img_model_title_256.load_state_dict(torch.load(\"checkpoints/img-model-title-256-epoch-5.pth\"))\n",
    "img_model_title_256.to((f'cuda:{img_model_title_256.device_ids[0]}'));\n",
    "img_model_title_256.eval();\n",
    "txt_model_title_256 = EmbeddingNetwork(256)\n",
    "txt_model_title_256 = nn.DataParallel(txt_model_title_256, device_ids=[1])\n",
    "txt_model_title_256.load_state_dict(torch.load(\"checkpoints/txt-model-title-256-epoch-5.pth\"))\n",
    "txt_model_title_256.to((f'cuda:{txt_model_title_256.device_ids[0]}'));\n",
    "txt_model_title_256.eval();\n",
    "\n",
    "#im2instructions 256\n",
    "img_model_instructions_256 = EmbeddingNetwork(256)\n",
    "img_model_instructions_256 = nn.DataParallel(img_model_instructions_256, device_ids=[1])\n",
    "img_model_instructions_256.load_state_dict(torch.load(\"checkpoints/img-model-instructions-256-epoch-5.pth\"))\n",
    "img_model_instructions_256.to((f'cuda:{img_model_instructions_256.device_ids[0]}'));\n",
    "img_model_instructions_256.eval();\n",
    "txt_model_instructions_256 = EmbeddingNetwork(256)\n",
    "txt_model_instructions_256 = nn.DataParallel(txt_model_instructions_256, device_ids=[1])\n",
    "txt_model_instructions_256.load_state_dict(torch.load(\"checkpoints/txt-model-instructions-256-epoch-5.pth\"))\n",
    "txt_model_instructions_256.to((f'cuda:{txt_model_instructions_256.device_ids[0]}'));\n",
    "txt_model_instructions_256.eval();\n",
    "\n",
    "#im2ingredients 256\n",
    "img_model_ingredients_256 = EmbeddingNetwork(256)\n",
    "img_model_ingredients_256 = nn.DataParallel(img_model_ingredients_256, device_ids=[1])\n",
    "img_model_ingredients_256.load_state_dict(torch.load(\"checkpoints/img-model-ingredients-256-epoch-5.pth\"))\n",
    "img_model_ingredients_256.to((f'cuda:{img_model_full_256.device_ids[0]}'));\n",
    "img_model_ingredients_256.eval();\n",
    "txt_model_ingredients_256 = EmbeddingNetwork(256)\n",
    "txt_model_ingredients_256 = nn.DataParallel(txt_model_ingredients_256, device_ids=[1])\n",
    "txt_model_ingredients_256.load_state_dict(torch.load(\"checkpoints/txt-model-ingredients-256-epoch-5.pth\"))\n",
    "txt_model_ingredients_256.to((f'cuda:{txt_model_ingredients_256.device_ids[0]}'));\n",
    "txt_model_ingredients_256.eval();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Full recipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1PP2zePpLWWQ",
    "outputId": "5bc51443-60d9-44bb-90b5-847c1e1dd23f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2recipe for dims = 256 and sample = 1000\n",
      "Mean median 6.95\n",
      "Recall {1: 0.2076, 5: 0.4608, 10: 0.571}\n",
      "Running im2recipe for dims = 256 and sample = 10000\n",
      "Mean median 59.2\n",
      "Recall {1: 0.05, 5: 0.16153, 10: 0.24162}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 256))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 256))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_full_256(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_full_256(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2recipe and recipe2im\n",
    "print(\"Running im2recipe for dims = 256 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2recipe for dims = 256 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dvv2Xs86LWWQ",
    "outputId": "ecc25cc9-529e-468e-e3a1-7395411303ca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2title for dims = 256 and sample = 1000\n",
      "Mean median 2.0\n",
      "Recall {1: 0.45620000000000005, 5: 0.7178, 10: 0.7921000000000001}\n",
      "Running im2title for dims = 256 and sample = 10000\n",
      "Mean median 9.1\n",
      "Recall {1: 0.19510000000000002, 5: 0.41746999999999995, 10: 0.52024}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 256))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 256))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_title_256(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_title_256(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2title and title2im\n",
    "print(\"Running im2title for dims = 256 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2title for dims = 256 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Ingredients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nARtvbqcLWWR",
    "outputId": "cc8b627f-75e8-40bf-a01c-8b782a51f8a6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2ingredients for dims = 256 and sample = 1000\n",
      "Mean median 13.3\n",
      "Recall {1: 0.09599999999999999, 5: 0.2969, 10: 0.4372}\n",
      "Running im2ingredients for dims = 256 and sample = 10000\n",
      "Mean median 127.05\n",
      "Recall {1: 0.015189999999999999, 5: 0.06169, 10: 0.10497000000000001}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 256))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 256))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_ingredients_256(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_ingredients_256(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2ingredients and ingredients2im\n",
    "print(\"Running im2ingredients for dims = 256 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2ingredients for dims = 256 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instructions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7O98Q2NQLWWR",
    "outputId": "ce59d651-dcda-4840-8795-609f1ef72ce2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2instructions for dims = 256 and sample = 1000\n",
      "Mean median 13.7\n",
      "Recall {1: 0.1041, 5: 0.3077, 10: 0.44000000000000006}\n",
      "Running im2instructions for dims = 256 and sample = 10000\n",
      "Mean median 129.0\n",
      "Recall {1: 0.01961, 5: 0.07319, 10: 0.12057999999999999}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 256))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 256))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_instructions_256(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_instructions_256(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2instructions and instructions2im\n",
    "print(\"Running im2instructions for dims = 256 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2instructions for dims = 256 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "fa8a4ffb"
   },
   "outputs": [],
   "source": [
    "# im2recipe 128\n",
    "img_model_full_128 = EmbeddingNetwork(128)\n",
    "img_model_full_128 = nn.DataParallel(img_model_full_128, device_ids=[1])\n",
    "img_model_full_128.load_state_dict(torch.load(\"checkpoints/img-model-full-128-epoch-10.pth\", map_location = \"cuda:1\"))\n",
    "img_model_full_128.to((f'cuda:{img_model_full_128.device_ids[0]}'));\n",
    "img_model_full_128.eval();\n",
    "txt_model_full_128 = EmbeddingNetwork(128)\n",
    "txt_model_full_128 = nn.DataParallel(txt_model_full_128, device_ids=[1])\n",
    "txt_model_full_128.load_state_dict(torch.load(\"checkpoints/txt-model-full-128-epoch-10.pth\", map_location = \"cuda:1\"))\n",
    "txt_model_full_128.to((f'cuda:{txt_model_full_128.device_ids[0]}'));\n",
    "txt_model_full_128.eval();\n",
    "\n",
    "#im2title 128\n",
    "img_model_title_128 = EmbeddingNetwork(128)\n",
    "img_model_title_128 = nn.DataParallel(img_model_title_128, device_ids=[1])\n",
    "img_model_title_128.load_state_dict(torch.load(\"checkpoints/img-model-title-128-epoch-5.pth\", map_location = \"cuda:1\"))\n",
    "img_model_title_128.to((f'cuda:{img_model_title_128.device_ids[0]}'));\n",
    "img_model_title_128.eval();\n",
    "txt_model_title_128 = EmbeddingNetwork(128)\n",
    "txt_model_title_128 = nn.DataParallel(txt_model_title_128, device_ids=[1])\n",
    "txt_model_title_128.load_state_dict(torch.load(\"checkpoints/txt-model-title-128-epoch-5.pth\", map_location = \"cuda:1\"))\n",
    "txt_model_title_128.to((f'cuda:{txt_model_title_128.device_ids[0]}'));\n",
    "txt_model_title_128.eval();\n",
    "\n",
    "#im2instructions 128\n",
    "img_model_instructions_128 = EmbeddingNetwork(128)\n",
    "img_model_instructions_128 = nn.DataParallel(img_model_instructions_128, device_ids=[1])\n",
    "img_model_instructions_128.load_state_dict(torch.load(\"checkpoints/img-model-instructions-128-epoch-5.pth\", map_location = \"cuda:1\"))\n",
    "img_model_instructions_128.to((f'cuda:{img_model_instructions_128.device_ids[0]}'));\n",
    "img_model_instructions_128.eval();\n",
    "txt_model_instructions_128 = EmbeddingNetwork(128)\n",
    "txt_model_instructions_128 = nn.DataParallel(txt_model_instructions_128, device_ids=[1])\n",
    "txt_model_instructions_128.load_state_dict(torch.load(\"checkpoints/txt-model-instructions-128-epoch-5.pth\", map_location = \"cuda:1\"))\n",
    "txt_model_instructions_128.to((f'cuda:{txt_model_instructions_128.device_ids[0]}'));\n",
    "txt_model_instructions_128.eval();\n",
    "\n",
    "#im2ingredients 128\n",
    "img_model_ingredients_128 = EmbeddingNetwork(128)\n",
    "img_model_ingredients_128 = nn.DataParallel(img_model_ingredients_128, device_ids=[1])\n",
    "img_model_ingredients_128.load_state_dict(torch.load(\"checkpoints/img-model-ingredients-128-epoch-5.pth\", map_location = \"cuda:1\"))\n",
    "img_model_ingredients_128.to((f'cuda:{img_model_full_128.device_ids[0]}'));\n",
    "img_model_ingredients_128.eval();\n",
    "txt_model_ingredients_128 = EmbeddingNetwork(128)\n",
    "txt_model_ingredients_128 = nn.DataParallel(txt_model_ingredients_128, device_ids=[1])\n",
    "txt_model_ingredients_128.load_state_dict(torch.load(\"checkpoints/txt-model-ingredients-128-epoch-5.pth\", map_location = \"cuda:1\"))\n",
    "txt_model_ingredients_128.to((f'cuda:{txt_model_ingredients_128.device_ids[0]}'));\n",
    "txt_model_ingredients_128.eval();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Full recipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "4cf32bae",
    "outputId": "cea10112-e0c6-48e3-d07c-b8e20e61947d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2recipe for dims = 128 and sample = 1000\n",
      "Mean median 13.25\n",
      "Recall {1: 0.14609999999999998, 5: 0.3574, 10: 0.4633}\n",
      "Running im2recipe for dims = 128 and sample = 10000\n",
      "Mean median 122.0\n",
      "Recall {1: 0.031409999999999993, 5: 0.10732, 10: 0.16516000000000003}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 128))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 128))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_full_128(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_full_128(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2recipe and recipe2im\n",
    "print(\"Running im2recipe for dims = 128 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2recipe for dims = 128 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "20628288",
    "outputId": "718fdc4c-9dee-42de-b18a-bf51f33a2b8f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2title for dims = 128 and sample = 1000\n",
      "Mean median 2.0\n",
      "Recall {1: 0.40310000000000007, 5: 0.6633, 10: 0.7434999999999999}\n",
      "Running im2title for dims = 128 and sample = 10000\n",
      "Mean median 13.65\n",
      "Recall {1: 0.16215000000000002, 5: 0.36424000000000006, 10: 0.4618300000000001}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 128))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 128))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_title_128(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_title_128(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2title and title2im\n",
    "print(\"Running im2title for dims = 128 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2title for dims = 128 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingredients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "13f66231",
    "outputId": "02314dba-10af-4109-9845-673a9cdab1ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2ingredients for dims = 128 and sample = 1000\n",
      "Mean median 36.0\n",
      "Recall {1: 0.06910000000000001, 5: 0.2001, 10: 0.2903}\n",
      "Running im2ingredients for dims = 128 and sample = 10000\n",
      "Mean median 334.85\n",
      "Recall {1: 0.014300000000000002, 5: 0.05009, 10: 0.07991}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 128))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 128))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_ingredients_128(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_ingredients_128(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2ingredients and ingredients2im\n",
    "print(\"Running im2ingredients for dims = 128 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2ingredients for dims = 128 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instructions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "5ac0916c",
    "outputId": "5d5d9cf4-9708-4921-fd53-8388d275b7ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2instructions for dims = 128 and sample = 1000\n",
      "Mean median 46.65\n",
      "Recall {1: 0.06220000000000001, 5: 0.18119999999999997, 10: 0.2584000000000001}\n",
      "Running im2instructions for dims = 128 and sample = 10000\n",
      "Mean median 448.1\n",
      "Recall {1: 0.01051, 5: 0.04272, 10: 0.06999999999999999}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 128))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 128))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_instructions_128(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_instructions_128(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2instructions and instructions2im\n",
    "print(\"Running im2instructions for dims = 128 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2instructions for dims = 128 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "fa8a4ffb"
   },
   "outputs": [],
   "source": [
    "# im2recipe 64\n",
    "img_model_full_64 = EmbeddingNetwork(64)\n",
    "img_model_full_64 = nn.DataParallel(img_model_full_64, device_ids=[1])\n",
    "img_model_full_64.load_state_dict(torch.load(\"checkpoints/img-model-full-64-epoch-10.pth\"))\n",
    "img_model_full_64.to((f'cuda:{img_model_full_64.device_ids[0]}'));\n",
    "img_model_full_64.eval();\n",
    "txt_model_full_64 = EmbeddingNetwork(64)\n",
    "txt_model_full_64 = nn.DataParallel(txt_model_full_64, device_ids=[1])\n",
    "txt_model_full_64.load_state_dict(torch.load(\"checkpoints/txt-model-full-64-epoch-10.pth\"))\n",
    "txt_model_full_64.to((f'cuda:{txt_model_full_64.device_ids[0]}'));\n",
    "txt_model_full_64.eval();\n",
    "\n",
    "#im2title 64\n",
    "img_model_title_64 = EmbeddingNetwork(64)\n",
    "img_model_title_64 = nn.DataParallel(img_model_title_64, device_ids=[1])\n",
    "img_model_title_64.load_state_dict(torch.load(\"checkpoints/img-model-title-64-epoch-5.pth\"))\n",
    "img_model_title_64.to((f'cuda:{img_model_title_64.device_ids[0]}'));\n",
    "img_model_title_64.eval();\n",
    "txt_model_title_64 = EmbeddingNetwork(64)\n",
    "txt_model_title_64 = nn.DataParallel(txt_model_title_64, device_ids=[1])\n",
    "txt_model_title_64.load_state_dict(torch.load(\"checkpoints/txt-model-title-64-epoch-5.pth\"))\n",
    "txt_model_title_64.to((f'cuda:{txt_model_title_64.device_ids[0]}'));\n",
    "txt_model_title_64.eval();\n",
    "\n",
    "#im2instructions 64\n",
    "img_model_instructions_64 = EmbeddingNetwork(64)\n",
    "img_model_instructions_64 = nn.DataParallel(img_model_instructions_64, device_ids=[1])\n",
    "img_model_instructions_64.load_state_dict(torch.load(\"checkpoints/img-model-instructions-64-epoch-5.pth\"))\n",
    "img_model_instructions_64.to((f'cuda:{img_model_instructions_64.device_ids[0]}'));\n",
    "img_model_instructions_64.eval();\n",
    "txt_model_instructions_64 = EmbeddingNetwork(64)\n",
    "txt_model_instructions_64 = nn.DataParallel(txt_model_instructions_64, device_ids=[1])\n",
    "txt_model_instructions_64.load_state_dict(torch.load(\"checkpoints/txt-model-instructions-64-epoch-5.pth\"))\n",
    "txt_model_instructions_64.to((f'cuda:{txt_model_instructions_64.device_ids[0]}'));\n",
    "txt_model_instructions_64.eval();\n",
    "\n",
    "#im2ingredients 64\n",
    "img_model_ingredients_64 = EmbeddingNetwork(64)\n",
    "img_model_ingredients_64 = nn.DataParallel(img_model_ingredients_64, device_ids=[1])\n",
    "img_model_ingredients_64.load_state_dict(torch.load(\"checkpoints/img-model-ingredients-64-epoch-5.pth\"))\n",
    "img_model_ingredients_64.to((f'cuda:{img_model_full_64.device_ids[0]}'));\n",
    "img_model_ingredients_64.eval();\n",
    "txt_model_ingredients_64 = EmbeddingNetwork(64)\n",
    "txt_model_ingredients_64 = nn.DataParallel(txt_model_ingredients_64, device_ids=[1])\n",
    "txt_model_ingredients_64.load_state_dict(torch.load(\"checkpoints/txt-model-ingredients-64-epoch-5.pth\"))\n",
    "txt_model_ingredients_64.to((f'cuda:{txt_model_ingredients_64.device_ids[0]}'));\n",
    "txt_model_ingredients_64.eval();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Full recipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "4cf32bae",
    "outputId": "cea10112-e0c6-48e3-d07c-b8e20e61947d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2recipe for dims = 512 and sample = 1000\n",
      "Mean median 51.3\n",
      "Recall {1: 0.056499999999999995, 5: 0.1654, 10: 0.2392}\n",
      "Running im2recipe for dims = 512 and sample = 10000\n",
      "Mean median 510.75\n",
      "Recall {1: 0.009409999999999998, 5: 0.03598, 10: 0.05913}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 64))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 64))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_full_64(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_full_64(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2recipe and recipe2im\n",
    "print(\"Running im2recipe for dims = 64 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2recipe for dims = 64 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "20628288",
    "outputId": "718fdc4c-9dee-42de-b18a-bf51f33a2b8f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2title for dims = 512 and sample = 1000\n",
      "Mean median 4.9\n",
      "Recall {1: 0.27979999999999994, 5: 0.5205, 10: 0.6196999999999999}\n",
      "Running im2title for dims = 512 and sample = 10000\n",
      "Mean median 38.7\n",
      "Recall {1: 0.09568999999999998, 5: 0.24392, 10: 0.32765999999999995}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 64))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 64))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_title_64(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_title_64(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2title and title2im\n",
    "print(\"Running im2title for dims = 64 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2title for dims = 64 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingredients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "13f66231",
    "outputId": "02314dba-10af-4109-9845-673a9cdab1ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2ingredients for dims = 512 and sample = 1000\n",
      "Mean median 84.6\n",
      "Recall {1: 0.0372, 5: 0.11200000000000002, 10: 0.1706}\n",
      "Running im2ingredients for dims = 512 and sample = 10000\n",
      "Mean median 828.8\n",
      "Recall {1: 0.00601, 5: 0.022809999999999997, 10: 0.038959999999999995}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 64))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 64))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_ingredients_64(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_ingredients_64(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2ingredients and ingredients2im\n",
    "print(\"Running im2ingredients for dims = 64 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2ingredients for dims = 64 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instructions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "5ac0916c",
    "outputId": "5d5d9cf4-9708-4921-fd53-8388d275b7ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2instructions for dims = 512 and sample = 1000\n",
      "Mean median 84.05\n",
      "Recall {1: 0.028300000000000002, 5: 0.09469999999999999, 10: 0.1502}\n",
      "Running im2instructions for dims = 512 and sample = 10000\n",
      "Mean median 835.5\n",
      "Recall {1: 0.005619999999999999, 5: 0.02037, 10: 0.033839999999999995}\n"
     ]
    }
   ],
   "source": [
    "img_val_nonlinear = np.zeros(shape = (len(img_val), 64))\n",
    "text_val_nonlinear = np.zeros(shape = (len(img_val), 64))\n",
    "\n",
    "for i in range(len(img_val)):\n",
    "    img_val_nonlinear[i] = img_model_instructions_64(torch.Tensor(np.expand_dims(img_val[i], 0))).cpu().detach().numpy()\n",
    "    text_val_nonlinear[i] = txt_model_instructions_64(torch.Tensor(np.expand_dims(text_val[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2instructions and instructions2im\n",
    "print(\"Running im2instructions for dims = 64 and sample = 1000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2instructions for dims = 64 and sample = 10000\")\n",
    "ranker(img_val_nonlinear, text_val_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6fa626f8"
   },
   "source": [
    "### Evaluation and Ablation Studies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "b30c8a2a"
   },
   "source": [
    " We can see that dimensions = 512 has a better performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1MWUKqEeLWWR"
   },
   "outputs": [],
   "source": [
    "# im2recipe 512\n",
    "img_model_full_512 = EmbeddingNetwork(512)\n",
    "img_model_full_512 = nn.DataParallel(img_model_full_512, device_ids=[1])\n",
    "img_model_full_512.load_state_dict(torch.load(\"checkpoints/img-model-full-512-epoch-10.pth\"))\n",
    "img_model_full_512.to((f'cuda:{img_model_full_512.device_ids[0]}'));\n",
    "img_model_full_512.eval();\n",
    "txt_model_full_512 = EmbeddingNetwork(512)\n",
    "txt_model_full_512 = nn.DataParallel(txt_model_full_512, device_ids=[1])\n",
    "txt_model_full_512.load_state_dict(torch.load(\"checkpoints/txt-model-full-512-epoch-10.pth\"))\n",
    "txt_model_full_512.to((f'cuda:{txt_model_full_512.device_ids[0]}'));\n",
    "txt_model_full_512.eval();\n",
    "\n",
    "#im2title 512\n",
    "img_model_title_512 = EmbeddingNetwork(512)\n",
    "img_model_title_512 = nn.DataParallel(img_model_title_512, device_ids=[1])\n",
    "img_model_title_512.load_state_dict(torch.load(\"checkpoints/img-model-title-512-epoch-5.pth\"))\n",
    "img_model_title_512.to((f'cuda:{img_model_title_512.device_ids[0]}'));\n",
    "img_model_title_512.eval();\n",
    "txt_model_title_512 = EmbeddingNetwork(512)\n",
    "txt_model_title_512 = nn.DataParallel(txt_model_title_512, device_ids=[1])\n",
    "txt_model_title_512.load_state_dict(torch.load(\"checkpoints/txt-model-title-512-epoch-5.pth\"))\n",
    "txt_model_title_512.to((f'cuda:{txt_model_title_512.device_ids[0]}'));\n",
    "txt_model_title_512.eval();\n",
    "\n",
    "#im2instructions 512\n",
    "img_model_instructions_512 = EmbeddingNetwork(512)\n",
    "img_model_instructions_512 = nn.DataParallel(img_model_instructions_512, device_ids=[1])\n",
    "img_model_instructions_512.load_state_dict(torch.load(\"checkpoints/img-model-instructions-512-epoch-5.pth\"))\n",
    "img_model_instructions_512.to((f'cuda:{img_model_instructions_512.device_ids[0]}'));\n",
    "img_model_instructions_512.eval();\n",
    "txt_model_instructions_512 = EmbeddingNetwork(512)\n",
    "txt_model_instructions_512 = nn.DataParallel(txt_model_instructions_512, device_ids=[1])\n",
    "txt_model_instructions_512.load_state_dict(torch.load(\"checkpoints/txt-model-instructions-512-epoch-5.pth\"))\n",
    "txt_model_instructions_512.to((f'cuda:{txt_model_instructions_512.device_ids[0]}'));\n",
    "txt_model_instructions_512.eval();\n",
    "\n",
    "#im2ingredients 512\n",
    "img_model_ingredients_512 = EmbeddingNetwork(512)\n",
    "img_model_ingredients_512 = nn.DataParallel(img_model_ingredients_512, device_ids=[1])\n",
    "img_model_ingredients_512.load_state_dict(torch.load(\"checkpoints/img-model-ingredients-512-epoch-5.pth\"))\n",
    "img_model_ingredients_512.to((f'cuda:{img_model_full_512.device_ids[0]}'));\n",
    "img_model_ingredients_512.eval();\n",
    "txt_model_ingredients_512 = EmbeddingNetwork(512)\n",
    "txt_model_ingredients_512 = nn.DataParallel(txt_model_ingredients_512, device_ids=[1])\n",
    "txt_model_ingredients_512.load_state_dict(torch.load(\"checkpoints/txt-model-ingredients-512-epoch-5.pth\"))\n",
    "txt_model_ingredients_512.to((f'cuda:{txt_model_ingredients_512.device_ids[0]}'));\n",
    "txt_model_ingredients_512.eval();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Full recipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mmYQP27-LWWR",
    "outputId": "20fd1bff-b618-49f8-eeb4-b6f954f35232"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2recipe for dims = 512 and sample = 1000\n",
      "Mean median 5.2\n",
      "Recall {1: 0.23480000000000004, 5: 0.5183000000000001, 10: 0.6353}\n",
      "Running im2recipe for dims = 512 and sample = 10000\n",
      "Mean median 43.1\n",
      "Recall {1: 0.06054999999999999, 5: 0.18576, 10: 0.27418}\n"
     ]
    }
   ],
   "source": [
    "img_test_nonlinear = np.zeros(shape = (len(img_test), 512))\n",
    "text_test_nonlinear = np.zeros(shape = (len(img_test), 512))\n",
    "\n",
    "for i in range(len(img_test)):\n",
    "    img_test_nonlinear[i] = img_model_full_512(torch.Tensor(np.expand_dims(img_test[i], 0))).cpu().detach().numpy()\n",
    "    text_test_nonlinear[i] = txt_model_full_512(torch.Tensor(np.expand_dims(text_test[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2recipe and recipe2im\n",
    "print(\"Running im2recipe for dims = 512 and sample = 1000\")\n",
    "ranker(img_test_nonlinear, text_test_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2recipe for dims = 512 and sample = 10000\")\n",
    "ranker(img_test_nonlinear, text_test_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "O_SmRhf-LWWS",
    "outputId": "32359cd4-96fe-47b0-8ca2-99ef148eb36b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2title for dims = 512 and sample = 1000\n",
      "Mean median 24.6\n",
      "Recall {1: 0.12349999999999998, 5: 0.29239999999999994, 10: 0.3778}\n",
      "Running im2title for dims = 512 and sample = 10000\n",
      "Mean median 247.35\n",
      "Recall {1: 0.03185, 5: 0.09637999999999998, 10: 0.14421}\n"
     ]
    }
   ],
   "source": [
    "img_test_nonlinear = np.zeros(shape = (len(img_test), 512))\n",
    "text_test_nonlinear = np.zeros(shape = (len(img_test), 512))\n",
    "\n",
    "for i in range(len(img_test)):\n",
    "    img_test_nonlinear[i] = img_model_title_512(torch.Tensor(np.expand_dims(img_test[i], 0))).cpu().detach().numpy()\n",
    "    text_test_nonlinear[i] = txt_model_title_512(torch.Tensor(np.expand_dims(title_test[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2title and title2im\n",
    "print(\"Running im2title for dims = 512 and sample = 1000\")\n",
    "ranker(img_test_nonlinear, text_test_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2title for dims = 512 and sample = 10000\")\n",
    "ranker(img_test_nonlinear, text_test_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingredients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zdH1bFQ_LWWS",
    "outputId": "ef7b6bcf-c96c-4b42-beab-5bae641f772b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2ingredients for dims = 512 and sample = 1000\n",
      "Mean median 15.2\n",
      "Recall {1: 0.08979999999999998, 5: 0.2829, 10: 0.41550000000000004}\n",
      "Running im2ingredients for dims = 512 and sample = 10000\n",
      "Mean median 141.25\n",
      "Recall {1: 0.014410000000000001, 5: 0.05796, 10: 0.10085999999999999}\n"
     ]
    }
   ],
   "source": [
    "img_test_nonlinear = np.zeros(shape = (len(img_test), 512))\n",
    "text_test_nonlinear = np.zeros(shape = (len(img_test), 512))\n",
    "\n",
    "for i in range(len(img_test)):\n",
    "    img_test_nonlinear[i] = img_model_ingredients_512(torch.Tensor(np.expand_dims(img_test[i], 0))).cpu().detach().numpy()\n",
    "    text_test_nonlinear[i] = txt_model_ingredients_512(torch.Tensor(np.expand_dims(ingredients_test[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2ingredients and ingredients2im\n",
    "print(\"Running im2ingredients for dims = 512 and sample = 1000\")\n",
    "ranker(img_test_nonlinear, text_test_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2ingredients for dims = 512 and sample = 10000\")\n",
    "ranker(img_test_nonlinear, text_test_nonlinear, 10000, \"image\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instructions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QEmHUX9sLWWS",
    "outputId": "223fe7bf-35d6-458d-a5c0-e9474716f9c4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running im2instructions for dims = 512 and sample = 1000\n",
      "Mean median 21.85\n",
      "Recall {1: 0.077, 5: 0.2365, 10: 0.35269999999999996}\n",
      "Running im2instructions for dims = 512 and sample = 10000\n",
      "Mean median 204.25\n",
      "Recall {1: 0.012629999999999999, 5: 0.04911, 10: 0.08324000000000001}\n"
     ]
    }
   ],
   "source": [
    "img_test_nonlinear = np.zeros(shape = (len(img_test), 512))\n",
    "text_test_nonlinear = np.zeros(shape = (len(img_test), 512))\n",
    "\n",
    "for i in range(len(img_test)):\n",
    "    img_test_nonlinear[i] = img_model_instructions_512(torch.Tensor(np.expand_dims(img_test[i], 0))).cpu().detach().numpy()\n",
    "    text_test_nonlinear[i] = txt_model_instructions_512(torch.Tensor(np.expand_dims(instructions_test[i], 0))).cpu().detach().numpy()\n",
    "\n",
    "# im2instructions and instructions2im\n",
    "print(\"Running im2instructions for dims = 512 and sample = 1000\")\n",
    "ranker(img_test_nonlinear, text_test_nonlinear, 1000, \"image\")\n",
    "print(\"Running im2instructions for dims = 512 and sample = 10000\")\n",
    "ranker(img_test_nonlinear, text_test_nonlinear, 10000, \"image\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}