vgg16-deeplabv3-SYNTHEIA-VIPER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install umap-learn\n",
    "# !pip uninstall umap -y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# End-End training and testing of the new pytorch-based VGG16-deeplabv3 architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/37893755/tensorflow-set-cuda-visible-devices-within-jupyter\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "# https://stackoverflow.com/questions/56008683/could-not-create-cudnn-handle-cudnn-status-internal-error\n",
    "import tensorflow as tf\n",
    "gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=0.95)\n",
    "config = tf.compat.v1.ConfigProto(gpu_options=gpu_options)\n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.compat.v1.Session(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.framework.ops import disable_eager_execution\n",
    "disable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "\n",
    "import wasserstein_utils\n",
    "import data_utils\n",
    "import losses\n",
    "import networks\n",
    "import deeplabv3 as dlv3\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "\n",
    "import data_utils\n",
    "import data_utils_new\n",
    "import losses\n",
    "\n",
    "from IPython import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/global/homes/f/fangjun/.conda/envs/myenv/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import umap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "backbone = 'vgg16'\n",
    "dataset = \"SYNTHIA_VIPER_COMBINED_13_0331\"\n",
    "\n",
    "# H x W x C\n",
    "img_shape = (512,1024,3)\n",
    "\n",
    "# 13 classes + void\n",
    "num_classes = 14\n",
    "\n",
    "batch_size=4\n",
    "\n",
    "do_training = False\n",
    "\n",
    "epochs=100000\n",
    "epoch_step=250\n",
    "\n",
    "num_projections=100\n",
    "\n",
    "source_data_dir = './processed-data/1024x512/13_classes/SYNTHIA/train/'\n",
    "cityscapes_data_dir = './processed-data/1024x512/13_classes/VIPER_COMBINED/train/'\n",
    "\n",
    "fn_w_dlv3 = \"weights/\" + dataset + \"/\" + backbone + \"_deeplabv3.h5\"\n",
    "fn_w_cls = \"weights/\" + dataset + \"/\" + backbone + \"_deeplabv3_classifier.h5\"\n",
    "\n",
    "fn_w_adapted_dlv3 = \"weights/\" + dataset + \"/\" + backbone +\"_deeplabv3_adapted.h5\"\n",
    "fn_w_adapted_cls = \"weights/\" + dataset + \"/\" + backbone + \"_deeplabv3_classifier_adapted.h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /global/homes/z/zuang99/.conda/envs/MAS/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:1666: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(losses)\n",
    "\n",
    "deeplabv3 = dlv3.deeplabv3(activation=None, \\\n",
    "                           backbone=backbone, \\\n",
    "                           num_classes=num_classes)\n",
    "\n",
    "X = deeplabv3.input\n",
    "Y = tf.keras.layers.Input(shape=(img_shape[0], img_shape[1], num_classes,), dtype='float32', name='label_input')\n",
    "\n",
    "C_in = tf.keras.layers.Input(shape=deeplabv3.layers[-1].output_shape[1:], dtype='float32', name='classifier_input')\n",
    "classifier = tf.keras.Model(C_in, networks.classifier_layers(C_in, num_classes = num_classes, activation='softmax'))\n",
    "\n",
    "# A combined model, giving the output of classifier(deeplabv3(X))\n",
    "combined = tf.keras.Model(X, classifier(deeplabv3(X)))\n",
    "combined.compile(loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False))\n",
    "\n",
    "# A model outputting hxwx1 labels for each image. Also useful to verify the\n",
    "# mIoU with Keras' built-in function. Will however also consider the 'ignore' class. \n",
    "combined_ = tf.keras.Model(X, tf.cast(tf.keras.backend.argmax(combined(X), axis=-1), 'float32'))\n",
    "combined_.compile(metrics=[tf.keras.metrics.MeanIoU(num_classes=num_classes)], loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False))\n",
    "\n",
    "# Set up training\n",
    "opt = tf.keras.optimizers.Adam(lr=1e-4, epsilon=1e-8, decay=1e-6)\n",
    "# opt = tf.keras.optimizers.SGD(lr=1e-4, momentum=.9)\n",
    "\n",
    "loss_function = losses.weighted_ce_loss(num_classes, 0)\n",
    "wce_loss = loss_function(Y, classifier(deeplabv3(X)), from_logits=False)\n",
    "\n",
    "# https://stackoverflow.com/questions/55434653/batch-normalization-doesnt-have-gradient-in-tensorflow-2-0\n",
    "params = deeplabv3.trainable_weights + classifier.trainable_weights\n",
    "\n",
    "updates = opt.get_updates(wce_loss, params)\n",
    "\n",
    "train = tf.keras.backend.function(inputs=[X,Y], outputs=[wce_loss], updates=updates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "deeplabv3.load_weights(fn_w_dlv3)\n",
    "classifier.load_weights(fn_w_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model weights\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "importlib.reload(data_utils)\n",
    "\n",
    "#do_training = True\n",
    "\n",
    "# Training on source domain\n",
    "if do_training == True:\n",
    "    try:\n",
    "        deeplabv3.load_weights(fn_w_dlv3)\n",
    "        classifier.load_weights(fn_w_cls)\n",
    "\n",
    "        print(\"Successfully loaded model. Continuing training.\")\n",
    "    except:\n",
    "        print(\"Could not load previous model weights. Is a new model present?\")\n",
    "        \n",
    "    start_time = time.time()\n",
    "\n",
    "    fig,ax = plt.subplots(1,figsize=(10,7))\n",
    "\n",
    "    loss_history = []\n",
    "\n",
    "    for itr in range(epochs):\n",
    "    # for itr in range(1):\n",
    "        source_train_data, source_train_labels = utils.sample_batch(source_data_dir, batch_size=batch_size, seed=itr)\n",
    "        source_train_labels = tf.keras.utils.to_categorical(source_train_labels, num_classes=num_classes)\n",
    "\n",
    "        loss_history.append(train(inputs=[source_train_data, source_train_labels ]))\n",
    "\n",
    "        if np.isnan(np.sum(loss_history[-1])):\n",
    "            print('ERROR. ENCOUNTERED NAN')\n",
    "            break\n",
    "\n",
    "        if itr%epoch_step == 0:\n",
    "            if itr != 0:\n",
    "                ax.clear()\n",
    "                ax.plot(np.asarray(loss_history))\n",
    "\n",
    "            ax.set_title(\"Training loss on source domain\")\n",
    "            ax.set_xlabel(\"Epoch\")\n",
    "            ax.set_ylabel(\"Loss\")\n",
    "\n",
    "            display.clear_output(wait=True)\n",
    "            display.display(plt.gcf())\n",
    "            time.sleep(1e-3)\n",
    "            \n",
    "        if itr % (epochs // 10) == 0 or itr == epochs - 1:\n",
    "            deeplabv3.save_weights(fn_w_dlv3)\n",
    "            classifier.save_weights(fn_w_cls)\n",
    "\n",
    "    training_time = time.time() - start_time\n",
    "else:\n",
    "    deeplabv3.load_weights(fn_w_dlv3)\n",
    "    classifier.load_weights(fn_w_cls)\n",
    "    print(\"Loaded model weights\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "road 0.0\n",
      "sidewalk 0.0\n",
      "building 0.0\n",
      "traffic light 0.0\n",
      "traffic sign 0.0\n",
      "vegetation 0.0\n",
      "sky 0.0\n",
      "person 0.0\n",
      "rider 0.0\n",
      "car 0.0\n",
      "bus 0.0\n",
      "motorcycle 0.0\n",
      "bicycle 0.0\n",
      "0.0\n",
      "Computed SYNTHIA_VIPER_COMBINED_13_0331 mIoU in 3038.3256697654724\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "importlib.reload(utils)\n",
    "\n",
    "import data_utils_new\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "source_cat_iou,source_miou = utils.compute_miou(source_data_dir, combined_, data_utils_new.label_ids_13, 0)\n",
    "\n",
    "for k in source_cat_iou:\n",
    "    print(k, source_cat_iou[k])\n",
    "print(source_miou)\n",
    "\n",
    "print('Computed ' + dataset + ' mIoU in', time.time() - start_time)\n",
    "\n",
    "# road 0.8559070065772992\n",
    "# sidewalk 0.9043721610536399\n",
    "# building 0.8381948427759653\n",
    "# wall 0.632677142340526\n",
    "# fence 0.699526703820766\n",
    "# pole 0.6073661307913504\n",
    "# traffic light 0.7258756809296298\n",
    "# traffic sign 0.6789546854786402\n",
    "# vegetation 0.8526590005209183\n",
    "# terrain 0.7779495588027171\n",
    "# sky 0.9530645387149239\n",
    "# person 0.7316948194592244\n",
    "# rider 0.809945935261575\n",
    "# car 0.5922120101790109\n",
    "# truck 0.9211600020912953\n",
    "# bus 0.9270433021188236\n",
    "# train 0.8659149467023691\n",
    "# motorcycle 0.6754852581619956\n",
    "# bicycle 0.5876167577346231\n",
    "# 0.7704010780797522\n",
    "# Computed GTA5 mIoU in 1953.6462309360504"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(utils)\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "cit_cat_iou,cit_miou = utils.compute_miou(cityscapes_data_dir, combined_, data_utils_new.label_ids_13, 0)\n",
    "\n",
    "for k in cit_cat_iou:\n",
    "    print(k, cit_cat_iou[k])\n",
    "print(cit_miou)\n",
    "\n",
    "print('Computed VIPER mIoU in', time.time() - start_time)\n",
    "\n",
    "# road 0.4538541445555259\n",
    "# sidewalk 0.17761519682584992\n",
    "# building 0.5960010633542256\n",
    "# wall 0.11064090337021892\n",
    "# fence 0.08225737548235189\n",
    "# pole 0.11701661235497808\n",
    "# traffic light 0.2131517695419058\n",
    "# traffic sign 0.085832471689785\n",
    "# vegetation 0.7016830692115976\n",
    "# terrain 0.05272756739697445\n",
    "# sky 0.4267235195286716\n",
    "# person 0.30570697364634025\n",
    "# rider 0.03181916445284627\n",
    "# car 0.4930313336164962\n",
    "# truck 0.05910168795933546\n",
    "# bus 0.052471835602942996\n",
    "# train 0.003952731749887269\n",
    "# motorcycle 0.07819570698033228\n",
    "# bicycle 0.0073530220468457285\n",
    "# 0.2131124289140585\n",
    "# Computed CITYSCAPES mIoU in 249.57955932617188"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import data_utils_new\n",
    "# importlib.reload(utils)\n",
    "\n",
    "# start_time = time.time()\n",
    "# means, _, ct = utils.learn_gaussians(source_data_dir, deeplabv3, combined, batch_size, data_utils_new.label_ids_13)\n",
    "# print(\"computed means in\", time.time() - start_time)\n",
    "\n",
    "# start_time = time.time()\n",
    "# means, covs, ct = utils.learn_gaussians(source_data_dir, deeplabv3, combined, batch_size, data_utils_new.label_ids_13, \\\n",
    "#                                   initial_means=means)\n",
    "# print(\"finished training gaussians in\", time.time() - start_time)\n",
    "\n",
    "# np.save(\"./extras/means_1024x512_\" + backbone + \"deeplabv3_\" + dataset + \".npy\", means)\n",
    "# np.save(\"./extras/covs_1024x512_\" + backbone + \"deeplabv3_\" + dataset + \".npy\", covs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(means.shape[0]):\n",
    "#     print('=====================================================')\n",
    "#     print(means[i])\n",
    "#     print(covs[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# means = np.load(\"./extras/means_1024x512_\" + backbone + \"deeplabv3_\" + dataset + \".npy\")\n",
    "# covs = np.load(\"./extras/covs_1024x512_\" + backbone + \"deeplabv3_\" + dataset + \".npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# means = np.nan_to_num(means)\n",
    "# means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Photo Version Replay Buffer Initialization\n",
    "import replay_buffer_utils as rb_utils\n",
    "\n",
    "start_time = time.time()\n",
    "source_domain_portion = 0.1\n",
    "replay_buffer = rb_utils.replay_buffer(source_data_dir, source_domain_portion)\n",
    "print(time.time() - start_time)\n",
    "print(replay_buffer.capacity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import umap.umap_ as umap\n",
    "# import matplotlib.pyplot as plt\n",
    "# # Generate data from the gmm model and plot it\n",
    "\n",
    "# start_time = time.time()\n",
    "\n",
    "# n_samples = np.ones(num_classes, dtype=int)\n",
    "# n_samples[0] = 0\n",
    "# n_samples *= 2000\n",
    "\n",
    "# xx, yy = utils.sample_from_gaussians(means, covs, n_samples=n_samples)\n",
    "\n",
    "# NUM_COLORS = num_classes\n",
    "\n",
    "# reducer = umap.UMAP()\n",
    "\n",
    "# umap_embedding = reducer.fit_transform(xx)\n",
    "\n",
    "# plt.figure(figsize=(16,14))\n",
    "# cm = plt.get_cmap('gist_rainbow')\n",
    "\n",
    "# shift = 1 / len(data_utils_new.label_ids_13.keys())\n",
    "# idx = 0\n",
    "# for label in data_utils_new.label_ids_13:\n",
    "#     ind = yy == data_utils_new.label_ids_13[label]\n",
    "    \n",
    "#     plt.scatter(umap_embedding[:,0][ind], umap_embedding[:,1][ind], label=label, \\\n",
    "#                 color=cm(1.*idx/NUM_COLORS))\n",
    "#     idx += 1\n",
    "\n",
    "# plt.title(\"Embedding scatter-plot\")\n",
    "# plt.legend()\n",
    "    \n",
    "# plt.show()\n",
    "\n",
    "# print(time.time() - start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(wasserstein_utils)\n",
    "\n",
    "#Z_s = tf.keras.layers.Input(shape=(img_shape[0], img_shape[1], num_classes,) )\n",
    "Z_s = tf.keras.layers.Input(shape=(img_shape[0], img_shape[1], 3,))\n",
    "Y_s = tf.keras.backend.placeholder(shape=(None, img_shape[0], img_shape[1], num_classes), dtype='float32') #labels of input images oneHot\n",
    "lambda2 = 5e-1\n",
    "\n",
    "loss_function = losses.weighted_ce_loss(num_classes, 0)\n",
    "#wce_loss = loss_function(Y_s, classifier(Z_s), from_logits=False)\n",
    "wce_loss = loss_function(Y_s, classifier(deeplabv3(Z_s)), from_logits=False)\n",
    "\n",
    "\n",
    "# Wasserstein matcing loss\n",
    "theta = tf.keras.backend.placeholder(shape = (num_projections, num_classes), dtype='float32')\n",
    "#matching_loss = wasserstein_utils.sWasserstein_hd(deeplabv3(X), Z_s, theta, nclass=num_classes, Cp=None, Cq=None,)\n",
    "matching_loss = wasserstein_utils.sWasserstein_hd(deeplabv3(X), deeplabv3(Z_s), theta, nclass=num_classes, Cp=None, Cq=None,)\n",
    "\n",
    "# Overall loss is a weighted combination of the two losses\n",
    "total_loss = wce_loss + lambda2*matching_loss\n",
    "\n",
    "params = deeplabv3.trainable_weights + classifier.trainable_weights\n",
    "\n",
    "# Optimizer and training setup\n",
    "opt = tf.keras.optimizers.Adam(lr=1e-4, epsilon=1e-6, decay=1e-6)\n",
    "\n",
    "updates = opt.get_updates(total_loss, params)\n",
    "train = tf.keras.backend.function(inputs=[X,Z_s,Y_s,theta], outputs=[total_loss, wce_loss, matching_loss], updates=updates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = []\n",
    "target_miou = []\n",
    "\n",
    "deeplabv3.load_weights(fn_w_dlv3)\n",
    "classifier.load_weights(fn_w_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax=plt.subplots(2,figsize=(15,10))\n",
    "\n",
    "batch_size = 1\n",
    "\n",
    "thres = .99\n",
    "#epochs2 = 50000\n",
    "epochs2 = 2000\n",
    "epochstep = 100\n",
    "\n",
    "for itr in range(epochs2):\n",
    "    \n",
    "    if itr%100 == 0:\n",
    "        print('Iteration: {}'.format(itr))\n",
    "    \n",
    "    target_train_data, target_train_labels = utils.sample_batch(cityscapes_data_dir, batch_size=batch_size, seed=itr)\n",
    "\n",
    "    # make sure the #samples from gaussians match the distribution of the labels\n",
    "    n_samples = np.zeros(num_classes, dtype=int)\n",
    "    cls, ns = np.unique(target_train_labels, return_counts=True)\n",
    "    for i in range(len(cls)):\n",
    "        if cls[i] == 0:\n",
    "            continue\n",
    "        n_samples[cls[i]] = ns[i]\n",
    "\n",
    "    if np.sum(n_samples) % np.prod(img_shape) != 0:\n",
    "        remaining = np.prod(img_shape[:-1]) - np.sum(n_samples) % np.prod(img_shape[:-1])\n",
    "\n",
    "        aux = np.copy(n_samples) / np.sum(n_samples)\n",
    "        aux *= remaining\n",
    "        aux = np.floor(aux).astype('int')\n",
    "        \n",
    "        n_samples += aux\n",
    "\n",
    "        # in case there are extra samples left, dump them on the highest represented class\n",
    "        n_samples[np.argmax(n_samples)] += remaining - np.sum(aux)\n",
    "\n",
    "###   replay buffer statistics version\n",
    "#     Yembed,Yembedlabels = utils.sample_from_gaussians(means, covs, n_samples = n_samples)\n",
    "#     Yembed = Yembed.reshape(-1, img_shape[0], img_shape[1], num_classes)\n",
    "#     Yembedlabels = Yembedlabels.reshape(-1, img_shape[0], img_shape[1])\n",
    "\n",
    "#     Yembedlabels = tf.keras.utils.to_categorical(Yembedlabels, num_classes=num_classes)\n",
    "\n",
    "#     theta_instance = tf.keras.backend.variable(wasserstein_utils.generateTheta(num_projections,num_classes))\n",
    "#     loss.append(train(inputs=[target_train_data, Yembed, Yembedlabels, theta_instance]))\n",
    "###   statistics version ends here\n",
    "\n",
    "\n",
    "###   replay buffer photo version\n",
    "    X, Y = rb_utils.random_sample_replay(replay_buffer, 1)\n",
    "    Y = tf.keras.utils.to_categorical(Y, num_classes=num_classes)\n",
    "    theta_instance = tf.keras.backend.variable(wasserstein_utils.generateTheta(num_projections,num_classes))\n",
    "    # print(X.shape)\n",
    "    # print(Y.shape)\n",
    "    loss.append(train(inputs=[target_train_data, X, Y, theta_instance]))\n",
    "###   replay buffer photo version ends here\n",
    "\n",
    "    target_miou.append(combined_.evaluate(target_train_data, target_train_labels, verbose=False)[-1] * 20/19)\n",
    "    \n",
    "    if itr%epochstep==0:\n",
    "        # Debug info. First, the mIoU. Second, the categorical CE loss (ignoring class weights and containing) \n",
    "        # the ignore class\n",
    "        if itr != 0:\n",
    "            ax[0].clear()\n",
    "            \n",
    "            ll = np.asarray(loss)\n",
    "            ax[0].plot(ll[:,0], label='total loss')\n",
    "            ax[0].plot(ll[:,1], label='ce loss')\n",
    "            ax[0].plot(ll[:,2] * lambda2, label='wasserstein loss')\n",
    "            ax[0].legend()\n",
    "            \n",
    "        ax[0].set_title(\"Loss\")\n",
    "        ax[0].set_xlabel(\"Epochs\")\n",
    "        ax[0].set_ylabel(\"Loss\")\n",
    "        \n",
    "        if itr != 0:\n",
    "            ax[1].clear()\n",
    "            ax[1].plot(np.asarray(target_miou))\n",
    "        \n",
    "        ax[1].set_title(\"MIOU on target domain\")\n",
    "        ax[1].set_xlabel(\"Epochs\")\n",
    "        ax[1].set_ylabel(\"Mean IOU\")\n",
    "        \n",
    "        display.display(plt.gcf())\n",
    "        display.clear_output(wait=True)\n",
    "        \n",
    "        time.sleep(1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import utils\n",
    "importlib.reload(utils)\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "source_cat_iou,source_miou = utils.compute_miou(source_data_dir, combined_, data_utils.label_ids_13, 0)\n",
    "\n",
    "for k in source_cat_iou:\n",
    "    print(k, source_cat_iou[k])\n",
    "print(source_miou)\n",
    "\n",
    "print('Computed ' + dataset + ' mIoU in', time.time() - start_time)\n",
    "\n",
    "# road 0.6675523660118785\n",
    "# sidewalk 0.28336359271439726\n",
    "# building 0.6076388775470897\n",
    "# wall 0.2672064026700072\n",
    "# fence 0.17232300484931942\n",
    "# pole 0.2735294951039704\n",
    "# traffic light 0.32147051090247314\n",
    "# traffic sign 0.07400171986539407\n",
    "# vegetation 0.5965591779805058\n",
    "# terrain 0.2889806932148324\n",
    "# sky 0.8603173835723591\n",
    "# person 0.3489449704700763\n",
    "# rider 0.17189785226832072\n",
    "# car 0.5761907340114261\n",
    "# truck 0.2955718142857747\n",
    "# bus 0.14318027819603177\n",
    "# train 0.033001835642511974\n",
    "# motorcycle 0.12385339561083183\n",
    "# bicycle 0.025308995240981697\n",
    "# 0.3226785842188517\n",
    "# Computed GTA5 mIoU in 1957.6215891838074"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "cit_cat_iou,cit_miou = utils.compute_miou(cityscapes_data_dir, combined_, data_utils.label_ids_13, 0)\n",
    "\n",
    "for k in cit_cat_iou:\n",
    "    print(k, cit_cat_iou[k])\n",
    "print(cit_miou)\n",
    "\n",
    "print('Computed VIPER mIoU in', time.time() - start_time)\n",
    "\n",
    "# road 0.7883116582422506\n",
    "# sidewalk 0.6215464735885322\n",
    "# building 0.751691007583865\n",
    "# wall 0.3993120030432491\n",
    "# fence 0.39173486089325144\n",
    "# pole 0.302509334397975\n",
    "# traffic light 0.3019573318279201\n",
    "# traffic sign 0.38603017233885695\n",
    "# vegetation 0.7792104073113252\n",
    "# terrain 0.484321694588526\n",
    "# sky 0.8323752669785053\n",
    "# person 0.5206867614087272\n",
    "# rider 0.2377222662760571\n",
    "# car 0.8020324193022697\n",
    "# truck 0.4754062370048565\n",
    "# bus 0.593789975462107\n",
    "# train 0.5966522990567177\n",
    "# motorcycle 0.3119814031424603\n",
    "# bicycle 0.47793033593638945\n",
    "# 0.5292211530728338\n",
    "# Computed CITYSCAPES mIoU in 232.43373942375183"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deeplabv3.save_weights(fn_w_adapted_dlv3)\n",
    "classifier.save_weights(fn_w_adapted_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deeplabv3.load_weights(fn_w_adapted_dlv3)\n",
    "classifier.load_weights(fn_w_adapted_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "MAS2",
   "language": "python",
   "name": "mas2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}