Saliency_system_kld_loss.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Saliency_system_kld_loss.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nilesh0109/CV2_SoSe_19/blob/master/Saliency_system_kld_loss.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IxwVzzyrpD2R",
        "colab_type": "code",
        "outputId": "ca58455e-4366-4294-ffd9-66f4fe0610a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import zipfile\n",
        "from google.colab import files, drive\n",
        "\n",
        "drive.mount('/content/drive', force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EK13Xnq_pG2L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "zip_ref = zipfile.ZipFile(\"/content/drive/My Drive/Colab Notebooks/CV2 exercies/Archive.zip\", 'r')\n",
        "zip_ref.extractall(\"tmp\")\n",
        "zip_ref.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EayiS3g6pJ8W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import tensorflow as tf\n",
        "import imageio\n",
        "from __future__ import division\n",
        "from skimage.transform import resize\n",
        "from sklearn.model_selection import KFold"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZBWzAZLlpLcj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ROOT= 'tmp'\n",
        "NUM_IMAGES = 1200\n",
        "VAL_NUM_IMAGES = 400\n",
        "\n",
        "def load_train_data():\n",
        "  training_img_directory = ROOT+'/data/train/images'\n",
        "  training_fixation_directory = ROOT+'/data/train/fixations'\n",
        "  val_img_directory = ROOT+'/data/val/images'\n",
        "  val_fixation_directory = ROOT+'/data/val/fixations'\n",
        "  train_imgs = np.zeros((NUM_IMAGES + VAL_NUM_IMAGES, 180, 320, 3), dtype=np.uint8)\n",
        "  train_fixations = np.zeros((NUM_IMAGES + VAL_NUM_IMAGES, 180, 320, 1), dtype=np.uint8)\n",
        "  \n",
        "  for i in range(1, NUM_IMAGES + 1):\n",
        "    img_file = os.path.join(training_img_directory, '{:04d}.jpg'.format(i))\n",
        "    fixation_file = os.path.join(training_fixation_directory, '{:04d}.jpg'.format(i))\n",
        "    train_imgs[i-1] = imageio.imread(img_file)\n",
        "    fixation = imageio.imread(fixation_file)\n",
        "    train_fixations[i-1] = np.expand_dims(fixation, -1) # adds singleton dimension so fixation size is (180,320,1)\n",
        "  \n",
        "  for j in range(i + 1, i + VAL_NUM_IMAGES + 1):\n",
        "    img_file = os.path.join(val_img_directory, '{:04d}.jpg'.format(j))\n",
        "    fixation_file = os.path.join(val_fixation_directory, '{:04d}.jpg'.format(j))\n",
        "    train_imgs[j-1] = imageio.imread(img_file)\n",
        "    fixation = imageio.imread(fixation_file)\n",
        "    train_fixations[j-1] = np.expand_dims(fixation, -1) # adds singleton dimension so fixation size is (180,320,1)\n",
        "    \n",
        "  return train_imgs, train_fixations\n",
        "\n",
        "# Generator function will output one (image, target) tuple at a time,\n",
        "# and shuffle the data for each new epoch\n",
        "def data_generator(imgs, targets):\n",
        "\twhile True: # produce new epochs forever\n",
        "\t\t# Shuffle the data for this epoch\n",
        "\t\tidx = np.arange(imgs.shape[0])\n",
        "\t\tnp.random.shuffle(idx)\n",
        "\n",
        "\t\timgs = imgs[idx]\n",
        "\t\ttargets = targets[idx]\n",
        "\t\tfor i in range(imgs.shape[0]):\n",
        "\t\t\tyield imgs[i], targets[i]\n",
        "\n",
        "def get_batch_from_generator(gen, batchsize):\n",
        "\tbatch_imgs = []\n",
        "\tbatch_fixations = []\n",
        "\tfor i in range(batchsize):\n",
        "\t\timg, target = gen.__next__()\n",
        "\t\tbatch_imgs.append(img)\n",
        "\t\tbatch_fixations.append(target)\n",
        "\treturn np.array(batch_imgs), np.array(batch_fixations)  \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rFi73W_n0rEm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#tf.reset_default_graph()\n",
        "\n",
        "'''\n",
        "Model class\n",
        "'''\n",
        "\n",
        "class saliencyModel:\n",
        "  def __init__(self, model_weights=None, learning_rate=1e-4, batch_size=32, num_epochs=100, prior_downsampling_factor=10):\n",
        "\n",
        "    self.lr_rate = learning_rate\n",
        "    self.batch_size = batch_size\n",
        "    self.num_epochs = num_epochs\n",
        "    self.prior_downsampling_factor = prior_downsampling_factor\n",
        "    self.input_h = 180\n",
        "    self.input_w = 320\n",
        "    self.prior_h = self.input_h /(2 * prior_downsampling_factor)\n",
        "    self.prior_w = self.input_w / (2 * prior_downsampling_factor)\n",
        "    self.reg_lambda = 1 / (self.prior_h * self.prior_w)\n",
        "    self.input_images_placeholder = tf.placeholder(tf.uint8, [None, self.input_h, self.input_w, 3])\n",
        "    self.target_images_placeholder = tf.placeholder(tf.uint8, [None, self.input_h, self.input_w, 1])\n",
        "    \n",
        "    if model_weights is not None:\n",
        "      self.load_weights(model_weights)\n",
        "    \n",
        "  def load_weights(self, model_weights):\n",
        "    vgg_weight_file = model_weights\n",
        "    self.weights = np.load(vgg_weight_file)\n",
        "  \n",
        "  def setup(self, mode= 'Train'):\n",
        "    \n",
        "    with tf.name_scope('preprocessing') as scope:\n",
        "      input_imgs = tf.image.convert_image_dtype(self.input_images_placeholder, tf.float32) * 255\n",
        "      fixations_normalized = tf.image.convert_image_dtype(self.target_images_placeholder, tf.float32)\n",
        "      mean = tf.constant([123.68 , 116.779 , 103.939], dtype = tf.float32, shape =[1,1,1,3], name ='img_mean')\n",
        "      imgs_normalized = input_imgs - mean\n",
        "\n",
        "    with tf.name_scope('conv1_1') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv1_1_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv1_1_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(imgs_normalized, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "\n",
        "    with tf.name_scope('conv1_2') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv1_2_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv1_2_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(act, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "\n",
        "    with tf.name_scope('pool1') as scope:\n",
        "      pool = tf.layers.max_pooling2d(act, pool_size=(2,2), strides=(2,2), padding='same')\n",
        "\n",
        "    with tf.name_scope('conv2_1') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv2_1_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv2_1_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(pool, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "\n",
        "    with tf.name_scope('conv2_2') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv2_2_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv2_2_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(act, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "\n",
        "    with tf.name_scope('pool2') as scope:\n",
        "      pool2 = tf.layers.max_pooling2d(act, pool_size=(2,2), strides=(2,2), padding='same')\n",
        "\n",
        "    with tf.name_scope('conv3_1') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv3_1_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv3_1_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(pool2, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "\n",
        "    with tf.name_scope('conv3_2') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv3_2_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv3_2_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(act, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "\n",
        "    with tf.name_scope('conv3_3') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv3_3_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv3_3_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(act, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "\n",
        "    with tf.name_scope('pool3') as scope:\n",
        "      pool3 = tf.layers.max_pooling2d(act, pool_size=(2,2), strides=(1,1), padding='same')\n",
        "\n",
        "    with tf.name_scope('conv4_1') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv4_1_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv4_1_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(pool3, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "      \n",
        "    with tf.name_scope('conv4_2') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv4_2_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv4_2_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(act, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "    \n",
        "    with tf.name_scope('conv4_3') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv4_3_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv4_3_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(act, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "      \n",
        "    with tf.name_scope('pool4') as scope:\n",
        "      pool4 = tf.layers.max_pooling2d(act, pool_size=(2,2), strides=(1,1), padding='same')\n",
        "    \n",
        "    with tf.name_scope('conv5_1') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv5_1_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv5_1_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(pool4, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "      \n",
        "    with tf.name_scope('conv5_2') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv5_2_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv5_2_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(act, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "    \n",
        "    with tf.name_scope('conv5_3') as scope:\n",
        "      kernel = tf.Variable(initial_value=self.weights['conv5_3_W'], trainable=False, name=\"weights\")\n",
        "      biases = tf.Variable(initial_value=self.weights['conv5_3_b'], trainable=False, name=\"biases\")\n",
        "      conv = tf.nn.conv2d(act, kernel, [1, 1, 1, 1], padding='SAME')\n",
        "      out = tf.nn.bias_add(conv, biases)\n",
        "      act = tf.nn.relu(out, name=scope)\n",
        "      conv5_3 = act\n",
        "      \n",
        "    with tf.name_scope('concat_featuremaps') as scope:\n",
        "      concatenated_feature_maps = tf.concat([pool2, pool3, pool4, conv5_3], axis=3)\n",
        "      print('IN_TRAINING_MODE is',mode =='Train')\n",
        "      regularized_feature_maps = tf.layers.dropout(concatenated_feature_maps, rate=0.5, training= mode =='Train')\n",
        "\n",
        "    with tf.name_scope('featuremaps_conv1') as scope:\n",
        "      my_regularizer = tf.contrib.layers.l2_regularizer(1e-5)\n",
        "      act_featuremaps_conv1 = tf.layers.conv2d(regularized_feature_maps, filters=64, kernel_size=(3,3), padding='SAME', activation=tf.nn.relu, name='featuremaps_conv1', kernel_regularizer = my_regularizer, reuse=tf.AUTO_REUSE)\n",
        "\n",
        "    with tf.name_scope('featuremaps_conv2') as scope:\n",
        "      act_featuremaps_conv2 = tf.layers.conv2d(act_featuremaps_conv1, filters=1, kernel_size=(1,1), activation=tf.nn.relu, name='featuremaps_conv2', kernel_regularizer = my_regularizer, reuse=tf.AUTO_REUSE)\n",
        "\n",
        "    with tf.name_scope('Learned_prior') as scope:\n",
        "      prior_shape = (1, self.prior_h, self.prior_w,1)\n",
        "      prior = tf.Variable(tf.ones(prior_shape), name=\"prior\", trainable=True)\n",
        "      upsampled_prior = tf.image.resize_bilinear(prior, size=act_featuremaps_conv2.shape[1:3])\n",
        "      saliency_mixed = tf.multiply(act_featuremaps_conv2, upsampled_prior)\n",
        "      saliency_raw = tf.nn.relu(saliency_mixed)\n",
        "      \n",
        "    with tf.name_scope('loss') as scope:\n",
        "      upsampled_saliency = tf.image.resize_bilinear(saliency_raw, size=fixations_normalized.shape[1:3])\n",
        "      # normalize saliency\n",
        "      max_value_per_image = tf.reduce_max(upsampled_saliency, axis=[1,2,3], keepdims=True)\n",
        "      predicted_saliency = (upsampled_saliency / max_value_per_image)\n",
        "      \n",
        "      # Loss function from Cornia et al. (2016) [with higher weight for salient pixels]\n",
        "#       alpha = 1.01\n",
        "#       weight = 1.0 / (alpha - fixations_normalized)\n",
        "#       loss = tf.losses.mean_squared_error(labels=fixations_normalized, \n",
        "#                         predictions=predicted_saliency, \n",
        "#                         weights=weight)\n",
        "\n",
        "      \n",
        "      \n",
        "      loss = self.get_kld(predicted_saliency ,fixations_normalized)\n",
        "      regularizer = tf.nn.l2_loss(1 - prior)\n",
        "      loss += tf.reduce_mean(self.reg_lambda * regularizer)\n",
        "      l2_loss = tf.losses.get_regularization_loss() \n",
        "      loss += l2_loss\n",
        "      \n",
        "    return predicted_saliency, loss\n",
        "  \n",
        "  def get_kld(self, p, g):\n",
        "      EPS = 1e-16\n",
        "      p_reshaped = tf.nn.softmax(tf.reshape(p, [tf.shape(p)[0], -1]))\n",
        "      g_reshaped = tf.nn.softmax(tf.reshape(g, [tf.shape(g)[0], -1]))\n",
        "      soft_p = tf.reshape(p_reshaped, tf.shape(p))\n",
        "      soft_g = tf.reshape(g_reshaped, tf.shape(g))\n",
        "      \n",
        "      kld = tf.reduce_sum(soft_g * tf.log( EPS + (soft_g / (EPS + soft_p) ) ))\n",
        "      return kld\n",
        "    \n",
        "  def train(self, t_imgs, t_fixations):\n",
        "    \n",
        "    pred_val ,loss_op_val = self.setup(mode='Test')\n",
        "    pred ,loss_op = self.setup(mode='Train')\n",
        "    training_loss = []\n",
        "    val_loss = []\n",
        "    \n",
        "    # Optimizer settings from Cornia et al. (2016) [except for decay]\n",
        "    optimizer = tf.train.MomentumOptimizer(learning_rate=self.lr_rate, momentum=0.9, use_nesterov=True)\n",
        "    minimize_op = optimizer.minimize(loss_op)\n",
        "    saver = tf.train.Saver()\n",
        "    \n",
        "    with tf.Session() as sess:\n",
        "      writer = tf.summary.FileWriter(logdir=\"events/\", graph=sess.graph)\n",
        "      sess.run(tf.global_variables_initializer())\n",
        "      saver.restore(sess, \"model/latest1-200\")\n",
        "      for epoch_ind in range(0, self.num_epochs, 5):\n",
        "        kf = KFold(n_splits=5)\n",
        "        for train_index, val_index in kf.split(t_imgs):\n",
        "          train_gen = data_generator(t_imgs[train_index], t_fixations[train_index])\n",
        "          val_gen = data_generator(t_imgs[val_index], t_fixations[val_index])\n",
        "          \n",
        "          num_batches = len(train_index) // self.batch_size\n",
        "          for b in range(num_batches):\n",
        "            batch_imgs, batch_fixations = get_batch_from_generator(train_gen, self.batch_size)\n",
        "            predication, batch_loss = sess.run([pred, loss_op], feed_dict={self.input_images_placeholder: batch_imgs, self.target_images_placeholder: batch_fixations})\n",
        "          \n",
        "          training_loss.append(batch_loss)\n",
        "          \n",
        "        if epoch_ind % 5 == 0:\n",
        "          #writer.add_summary(l_summary, global_step=b)\n",
        "          print('epoch {0} done: TRAINING batch loss {1}'.format(epoch_ind, batch_loss))\n",
        "            \n",
        "        if epoch_ind % 100 == 0:\n",
        "          val_batch_imgs, val_batch_fixations = get_batch_from_generator(val_gen, self.batch_size)\n",
        "          prediction_val, batch_loss_val = sess.run([pred_val, loss_op_val], feed_dict={self.input_images_placeholder: val_batch_imgs, self.target_images_placeholder: val_batch_fixations})\n",
        "          print('epoch {0} VALIDATION batch loss {1}'.format(epoch_ind, batch_loss_val))\n",
        "          val_loss.append(batch_loss_val)\n",
        "          \n",
        "        if epoch_ind % 100 == 0 and epoch_ind != 0:\n",
        "          save_path = saver.save(sess, 'model/latest1', global_step=epoch_ind)\n",
        "      save_path = saver.save(sess, 'model/latest1')\n",
        "      \n",
        "      \n",
        "  def test(self, test_imgs):\n",
        "    pred_saliency, l = self.setup(mode='Test')\n",
        "    saver = tf.train.Saver()\n",
        "    \n",
        "    with tf.Session() as sess:\n",
        "      #writer = tf.summary.FileWriter(logdir=\"./\", graph=sess.graph)\n",
        "      saver.restore(sess, 'model/latest1')\n",
        "      saliency = sess.run(pred_saliency, feed_dict={self.input_images_placeholder: test_imgs})\n",
        "      for i in range(len(test_imgs)):\n",
        "        print('saving images')\n",
        "        saliency_img = sess.run(tf.image.convert_image_dtype(saliency[i], tf.uint8))\n",
        "        imageio.imwrite(str(i)+'.jpg', saliency_img)\n",
        "    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GhmQ0g-1-GKV",
        "colab_type": "code",
        "outputId": "0e421a89-303f-4750-c992-8d79da2ceced",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "tf.reset_default_graph()\n",
        "\n",
        "train_imgs, train_fixations = load_train_data()\n",
        "\n",
        "my_model = saliencyModel(model_weights='vgg/VGG16/vgg16-conv-weights.npz', learning_rate=1e-1, num_epochs=1000, batch_size=32)\n",
        "\n",
        "my_model.train(train_imgs, train_fixations)\n",
        "#my_model.train(train_imgs, train_fixations)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0629 22:28:42.647910 139924442183552 deprecation.py:323] From <ipython-input-6-19eeb2da4856>:51: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.MaxPooling2D instead.\n",
            "W0629 22:28:43.391805 139924442183552 deprecation.py:323] From <ipython-input-6-19eeb2da4856>:143: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.dropout instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "IN_TRAINING_MODE is False\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0629 22:28:44.130783 139924442183552 lazy_loader.py:50] \n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "W0629 22:28:44.131863 139924442183552 deprecation.py:323] From <ipython-input-6-19eeb2da4856>:147: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.keras.layers.Conv2D` instead.\n",
            "W0629 22:28:44.269407 139924442183552 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "IN_TRAINING_MODE is True\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0629 22:28:47.590991 139924442183552 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use standard file APIs to check for files with this prefix.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch 0 done: TRAINING batch loss 0.3307967185974121\n",
            "epoch 0 VALIDATION batch loss 0.3172154724597931\n",
            "epoch 5 done: TRAINING batch loss 0.3114720284938812\n",
            "epoch 10 done: TRAINING batch loss 0.32900145649909973\n",
            "epoch 15 done: TRAINING batch loss 0.3079928159713745\n",
            "epoch 20 done: TRAINING batch loss 0.3131747543811798\n",
            "epoch 25 done: TRAINING batch loss 0.31188708543777466\n",
            "epoch 30 done: TRAINING batch loss 0.3228166103363037\n",
            "epoch 35 done: TRAINING batch loss 0.32937827706336975\n",
            "epoch 40 done: TRAINING batch loss 0.34835025668144226\n",
            "epoch 45 done: TRAINING batch loss 0.3306529223918915\n",
            "epoch 50 done: TRAINING batch loss 0.3326833248138428\n",
            "epoch 55 done: TRAINING batch loss 0.3126348555088043\n",
            "epoch 60 done: TRAINING batch loss 0.3118203282356262\n",
            "epoch 65 done: TRAINING batch loss 0.3352869153022766\n",
            "epoch 70 done: TRAINING batch loss 0.3268083333969116\n",
            "epoch 75 done: TRAINING batch loss 0.32368817925453186\n",
            "epoch 80 done: TRAINING batch loss 0.3426995873451233\n",
            "epoch 85 done: TRAINING batch loss 0.3455754518508911\n",
            "epoch 90 done: TRAINING batch loss 0.33459141850471497\n",
            "epoch 95 done: TRAINING batch loss 0.3392723500728607\n",
            "epoch 100 done: TRAINING batch loss 0.30913281440734863\n",
            "epoch 100 VALIDATION batch loss 0.33327361941337585\n",
            "epoch 105 done: TRAINING batch loss 0.3144087493419647\n",
            "epoch 110 done: TRAINING batch loss 0.3091817796230316\n",
            "epoch 115 done: TRAINING batch loss 0.31476232409477234\n",
            "epoch 120 done: TRAINING batch loss 0.2881356477737427\n",
            "epoch 125 done: TRAINING batch loss 0.2944941222667694\n",
            "epoch 130 done: TRAINING batch loss 0.294477254152298\n",
            "epoch 135 done: TRAINING batch loss 0.32922250032424927\n",
            "epoch 140 done: TRAINING batch loss 0.31114470958709717\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t1M8GSkh-GVC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.reset_default_graph()\n",
        "\n",
        "train_imgs, train_fixations = load_train_data()\n",
        "\n",
        "my_model = saliencyModel(model_weights='tmp/Model/VGG16/vgg16-conv-weights.npz', learning_rate=1e-1, num_batches=1001, batch_size=16)\n",
        "\n",
        "my_model.test(train_imgs[0:16])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ixmakTnm-GjF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}