run_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # or any {'0', '1', '2'}\n",
    "# 0: infos, warning, errors.\n",
    "# 1: warnings, errors.\n",
    "# 2: errors.\n",
    "# 3: none.\n",
    "import tensorflow as tf\n",
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "print(\"GPUs available:\", physical_devices)\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "import math\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "from ocml.datasets import build_ds, tfds_from_sampler, tf_from_tfds, zip_ds, preprocess_cifar10\n",
    "from ocml.evaluate import check_LLC, log_metrics\n",
    "from ocml.models import spectral_VGG, spectral_VGG_V2\n",
    "from ocml.plot import plot_preds_ood, plot_imgs_grid, plot_gan\n",
    "from ocml.priors import uniform_image, augment_cifar10, Cifar10_NDA\n",
    "from ocml.train import train, SH_KR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from types import SimpleNamespace\n",
    "import math\n",
    "\n",
    "\n",
    "def perc_to_margin(img_size, num_channels, perc, domain):\n",
    "    return perc * (img_size * img_size * num_channels * (domain[1] - domain[0]))**0.5\n",
    "\n",
    "def get_config(debug=False):\n",
    "  center_reduce = False\n",
    "  if center_reduce:\n",
    "    domain = [-2.1, 2.14]  # domain for centered-reduced images.\n",
    "  else:\n",
    "    domain = [0, 1]  # domain for canonical scale.\n",
    "  # heuristic of https://arxiv.org/abs/2206.06854\n",
    "  ratio_images = 0.1 / 100\n",
    "  ratio_pixels = 0.2 / 100\n",
    "  margin = perc_to_margin(32, 3, ratio_pixels, domain)\n",
    "  lbda = 1. / ratio_images  \n",
    "  print(f\"Margin={margin:.3f} Lambda={lbda:.3f}\")\n",
    "  dataset_name = os.environ.get(\"DATASET_NAME\", \"cifar10\")\n",
    "  in_class = int(os.environ.get(\"IN_CLASS\", 0))\n",
    "  config = SimpleNamespace(\n",
    "    dataset_name = dataset_name,\n",
    "    # Newton-Raphson.\n",
    "    maxiter = 32,\n",
    "    eta = 20.,\n",
    "    level_set = -margin,\n",
    "    batch_size = 128,\n",
    "    domain = domain,\n",
    "    center_reduce = center_reduce,\n",
    "    margin = margin,\n",
    "    lbda = lbda,\n",
    "    domain_clip = True,\n",
    "    deterministic = False,\n",
    "    negative_augmentation = False,\n",
    "    overshoot_boundary = True,\n",
    "    # architecture.\n",
    "    k_coef_lip = 1.,\n",
    "    strides = False,\n",
    "    spectral_dense = True,\n",
    "    pooling = True,\n",
    "    global_pooling = False,\n",
    "    groupsort = True,\n",
    "    conv_widths = [256, 256, 256],\n",
    "    dense_widths = [256, 256, 256],\n",
    "    # training.\n",
    "    in_labels = [in_class],\n",
    "    warmup_epochs = 10,\n",
    "    epochs_per_plot = 40,\n",
    "    resume_run = None\n",
    "  )\n",
    "  return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "debug = \"SANDBOX\" in os.environ\n",
    "config = get_config(debug)\n",
    "train_kwargs = {\n",
    "  'domain': config.domain,\n",
    "  'eta': config.eta,\n",
    "  'deterministic': config.deterministic,\n",
    "  'level_set': config.level_set,\n",
    "  'overshoot_boundary': config.overshoot_boundary\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.io as pio\n",
    "print(\"PLOTLY_RENDERER:\", pio.renderers.default)\n",
    "try:\n",
    "  import wandb\n",
    "  wandb.login()\n",
    "  wandb_available = True\n",
    "except ModuleNotFoundError as e:\n",
    "  print(e)\n",
    "  print(\"Wandb logs will be removed.\")\n",
    "  wandb_available = False\n",
    "plot_wandb = wandb_available and not debug  # Set to False to de-activate Wandb.\n",
    "if plot_wandb:\n",
    "  import wandb\n",
    "  group = os.environ.get(\"WANDB_GROUP\", \"sandbox_cifar10\")\n",
    "  wandb.init(project=\"csdfl_cifar10\", config=config.__dict__, group=group, save_code=True)\n",
    "else:\n",
    "  try:\n",
    "    wandb.finish()\n",
    "  except Exception as e:\n",
    "    print(e)\n",
    "\n",
    "train_kwargs['log_metrics_fn'] = partial(log_metrics, plot_wandb=plot_wandb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = (32, 32, 3)\n",
    "model = spectral_VGG_V2(input_shape, k_coef_lip=config.k_coef_lip, scale=1)\n",
    "\n",
    "loss_fn = SH_KR(config.margin, config.lbda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "build_ds = partial(build_ds, preprocess_fun=preprocess_cifar10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Produce and process dataset.\n",
    "domain_preprocess = None if config.center_reduce else config.domain\n",
    "p_dataset = build_ds(config.dataset_name, config.batch_size, in_labels=config.in_labels, domain=domain_preprocess)\n",
    "num_images = 50 * 1000\n",
    "epoch_length = math.ceil(num_images*len(config.in_labels)*(1/10) / config.batch_size) if not debug else 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_P = tf.reshape(tf_from_tfds(p_dataset.take(epoch_length)), shape=(-1,)+input_shape)\n",
    "p_dataset = augment_cifar10(p_dataset)\n",
    "X_test = tf_from_tfds(build_ds(config.dataset_name, config.batch_size, in_labels=config.in_labels, domain=domain_preprocess, split='test'))\n",
    "X_ood = tf_from_tfds(build_ds(config.dataset_name, config.batch_size, in_labels=config.in_labels, domain=domain_preprocess, split='ood'))\n",
    "print(f'TrainSize={len(X_P)} TestSize={len(X_test)} OODSize={len(X_ood)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create optimizer class.\n",
    "blocks = 4\n",
    "decay_steps = epoch_length*(config.warmup_epochs + config.epochs_per_plot*blocks)\n",
    "initial_learning_rate = 2.5e-4\n",
    "learning_rate =  tf.keras.optimizers.schedules.PolynomialDecay(\n",
    "    initial_learning_rate=initial_learning_rate, decay_steps=decay_steps,\n",
    "  end_learning_rate=initial_learning_rate/1000, power=1.)\n",
    "opt = tf.keras.optimizers.RMSprop(learning_rate=learning_rate)\n",
    "\n",
    "# Initialize the network.\n",
    "gen = tf.random.Generator.from_seed(random.randint(0, 1000))\n",
    "p_batch = next(iter(p_dataset))\n",
    "_ = model(p_batch, training=True)  # garbage forward.\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adversarial distribution.\n",
    "if config.negative_augmentation:\n",
    "  bs_1, bs_2 = config.batch_size // 2, config.batch_size - (config.batch_size // 2)\n",
    "  q_random = tfds_from_sampler(uniform_image, gen, bs_1, p_batch.shape[1:], domain=config.domain)\n",
    "  q_nda = Cifar10_NDA(bs_1, p_batch.shape[1:]).transform(gen, build_mnist(config.dataset_name, bs_2, in_labels=config.in_labels))\n",
    "  q_dataset = zip_ds(q_random, q_nda)\n",
    "else:\n",
    "  q_dataset = tfds_from_sampler(uniform_image, gen, config.batch_size, p_batch.shape[1:], domain=config.domain)\n",
    "Q0 = next(iter(q_dataset))\n",
    "plot_imgs_grid(Q0, 'X_ood.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot_imgs_grid(X_test, 'X_test.png')\n",
    "# plot_imgs_grid(X_ood, 'X_ood.png')\n",
    "# check_LLC(model, Q0, plot_wandb)\n",
    "plot_imgs_grid(X_P, 'X_P.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_imgs_grid(p_batch, 'p_batch.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if config.resume_run is not None:\n",
    "  from ocml.models import load_VGG_V2_from_run\n",
    "  print(f\"Resume run {config.resume_run}.\")\n",
    "  project, run_id = config.resume_run.split('/')\n",
    "  load_VGG_V2_from_run('algue', project, run_id, model)\n",
    "  _ = model(p_batch, training=True)  # garbage forward.\n",
    "  for layer in model.layers[:-1]:\n",
    "    layer.trainable = False\n",
    "  epoch = config.warmup_epochs + config.epochs_per_plot*2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 0\n",
    "for epoch in range(0, config.warmup_epochs):\n",
    "  print(f\"Epoch={epoch} LR={float(opt._decayed_lr(tf.float32)):.7f}\")\n",
    "  train(model, opt, loss_fn, gen, p_dataset, q_dataset, epoch_length, maxiter=0, **train_kwargs)\n",
    "plot_preds_ood(epoch, model, X_P, X_test, X_ood, plot_histogram=True, plot_wandb=plot_wandb)\n",
    "plot_gan(epoch, model, p_batch, Q0[:16], gen, maxiter=config.maxiter, **train_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_epoch = config.epochs_per_plot+epoch+1\n",
    "for epoch in range(epoch+1, end_epoch):\n",
    "  print(f\"Epoch={epoch} LR={float(opt._decayed_lr(tf.float32)):.7f}\")\n",
    "  train(model, opt, loss_fn, gen, p_dataset, q_dataset, epoch_length, maxiter=config.maxiter, **train_kwargs)\n",
    "  plot_histogram = (epoch+1 == end_epoch)\n",
    "  plot_preds_ood(epoch, model, X_P, X_test, X_ood, plot_histogram=plot_histogram, plot_wandb=plot_wandb)\n",
    "plot_gan(epoch, model, p_batch, Q0[:16], gen, maxiter=config.maxiter, **train_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_epoch = config.epochs_per_plot+epoch+1\n",
    "for epoch in range(epoch+1, end_epoch):\n",
    "  print(f\"Epoch={epoch} LR={float(opt._decayed_lr(tf.float32)):.7f}\")\n",
    "  train(model, opt, loss_fn, gen, p_dataset, q_dataset, epoch_length, maxiter=config.maxiter, **train_kwargs)\n",
    "  plot_histogram = (epoch+1 == end_epoch)\n",
    "  plot_preds_ood(epoch, model, X_P, X_test, X_ood, plot_histogram=plot_histogram, plot_wandb=plot_wandb)\n",
    "plot_gan(epoch, model, p_batch, Q0[:16], gen, maxiter=config.maxiter, **train_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ocml.evaluate import calibrate_accuracy\n",
    "\n",
    "def compute_certificates(model, config, X_train, X_test, X_ood):\n",
    "  # run XPs.\n",
    "  y_train    = model.predict(X_train, batch_size=256, verbose=0).flatten()\n",
    "  y_test     = model.predict(X_test, batch_size=256, verbose=0).flatten()\n",
    "  y_ood      = model.predict(X_ood, batch_size=256, verbose=0).flatten()\n",
    "  \n",
    "  msg = dict()\n",
    "  attacks_radii = [0, 8, 16, 36, 72, 144, 256]\n",
    "  for r in attacks_radii:\n",
    "    e = (r / 255) * (config.domain[1] - config.domain[0])\n",
    "    T_train, acc_train, roc_auc_train = calibrate_accuracy(y_train-e, y_ood+e)\n",
    "    T_test, acc_test, roc_auc_test = calibrate_accuracy(y_test-e, y_ood+e)\n",
    "    msg[f'certificate_r={r:}_train'] = roc_auc_train\n",
    "    msg[f'certificate_r={r:}_test'] = roc_auc_test\n",
    "  print(msg)\n",
    "  wandb.log(msg)\n",
    "  return msg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compute_certificates(model, config, X_P, X_test, X_ood)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "import tqdm\n",
    "\n",
    "if config.center_reduce:\n",
    "  bounds = [-2.1, 2.14]\n",
    "else:\n",
    "  bounds = [0, 1]\n",
    "\n",
    "def proj_l2_ball(x, x_0, eps):\n",
    "  n = x - x_0\n",
    "  l = tf.reduce_sum(n**2, axis=-1, keepdims=True)**0.5\n",
    "  l = tf.maximum(l, 1e-6 * eps)\n",
    "  factor = tf.where(l > eps, eps / l, 1.)\n",
    "  n = n * factor\n",
    "  x = x_0 + n\n",
    "  return x\n",
    "\n",
    "def random_ball(x_0, eps):\n",
    "  n = tf.random.normal((x_0.shape[0], x_0.shape[1]+1,))\n",
    "  l = tf.reduce_sum(n**2, axis=-1, keepdims=True)**0.5\n",
    "  n = n / tf.maximum(l, 1e-6)\n",
    "  n = n[:,:-1]  # drop last coordinate.\n",
    "  x = x_0 + n*eps\n",
    "  return x\n",
    "\n",
    "def normalize_grads(n):\n",
    "  l = tf.reduce_sum(n**2, axis=-1, keepdims=True)**0.5\n",
    "  l = tf.maximum(l, 1e-6)\n",
    "  n = n / l\n",
    "  return n\n",
    "\n",
    "def gd_step(x, x0, label, eps, model, step):\n",
    "  with tf.GradientTape(watch_accessed_variables=False) as tape:\n",
    "    tape.watch(x)\n",
    "    y = model(tf.reshape(x, (-1, 32, 32, 3)), training=False)\n",
    "  g = tape.batch_jacobian(y, x)\n",
    "  g = g[:,0,:]\n",
    "  g = normalize_grads(g)\n",
    "  x = x - step * label * g  # descent: decreases OOD score of OOD, increases of normal data\n",
    "  x = proj_l2_ball(x, x0, eps)\n",
    "  x = tf.clip_by_value(x, *bounds)\n",
    "  return x\n",
    "\n",
    "@tf.function\n",
    "def l2_pgd(model, x0, label, eps, attempts=1, random_start=True):\n",
    "  y_best = None\n",
    "  for attempt in range(attempts):\n",
    "    if random_start:\n",
    "      x = random_ball(x0, eps)\n",
    "    else:\n",
    "      x = x0\n",
    "    x = tf.clip_by_value(x, *bounds)\n",
    "    max_iter = 50\n",
    "    step = 0.025 * eps\n",
    "    for iter in range(max_iter):\n",
    "      x = gd_step(x, x0, label, eps, model, step)\n",
    "    delta = tf.reduce_mean(tf.reduce_sum((x - x0)**2, axis=-1)**0.5)\n",
    "    y = model(tf.reshape(x, (-1, 32, 32, 3)), training=False)\n",
    "    if y_best is None:\n",
    "      y_best = y\n",
    "    else:\n",
    "      y_min = tf.minimum(y, y_best)\n",
    "      y_max = tf.maximum(y, y_best)\n",
    "      y_best = tf.where(label[:,0] > 0., y_min, y_max)\n",
    "  return y_best, delta\n",
    "\n",
    "def l2_pgd_batch(model, images, labels, eps, batch_size):\n",
    "  scores = []\n",
    "  images = tf.reshape(images, (-1, batch_size, 3072))\n",
    "  labels = tf.reshape(labels, (-1, batch_size, 1))\n",
    "  deltas = [0.]\n",
    "  for x0, label in tqdm.tqdm(zip(images, labels)):\n",
    "    if eps == 0.:\n",
    "      x = tf.reshape(x0, shape=(-1, 32, 32, 3))\n",
    "      score = model(x, training=False)\n",
    "    else:\n",
    "      score, delta = l2_pgd(model, x0, label, eps)\n",
    "      deltas.append(delta)\n",
    "    scores.append(score.numpy().flatten())\n",
    "  scores = np.concatenate(np.array(scores), axis=0)\n",
    "  return scores, deltas\n",
    "\n",
    "def attack(model, config, X_train, X_test, X_ood, batch_size):\n",
    "  X_ood = np.random.permutation(X_ood)[:(len(X_ood) // batch_size)*batch_size]\n",
    "  X_test = np.random.permutation(X_test)[:(len(X_test) // batch_size)*batch_size]\n",
    "  images = tf.constant(np.concatenate([X_test, X_ood], axis=0))\n",
    "  labels = tf.concat([tf.ones((len(X_test),)), -tf.ones((len(X_ood),))], axis=0)\n",
    "  msg = dict()\n",
    "  attacks_radii = [0, 8, 16, 36, 72, 144, 255]\n",
    "  for r in attacks_radii:\n",
    "    e = (r / 255) * (config.domain[1] - config.domain[0])\n",
    "    scores, deltas = l2_pgd_batch(model, images, labels, eps=e, batch_size=batch_size)\n",
    "    deltas = np.mean(np.array(deltas))\n",
    "    print(\"deltas:\", deltas)\n",
    "    msg['deltas'] = deltas\n",
    "    roc_auc_test = roc_auc_score((labels+1)/2, scores)*100\n",
    "    print(f'empirical_r={r:}_test={roc_auc_test}%')\n",
    "    msg[f'empirical_r={r:}_test'] = roc_auc_test\n",
    "    print(scores[:10], scores[-10:])\n",
    "  print(msg)\n",
    "  wandb.log(msg)\n",
    "  return msg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# attack(model, config, X_P, X_test, X_ood, batch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if plot_wandb:\n",
    "  wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "92c440f6d82e79b41f51da7601df579ae39fc115954f77d4b816ac1ce1ea7f75"
   }
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "091c2994d1544a68ab89a94f725c5ed7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "24504d5cee27493f879777200b809d3c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "424b1f1a15ad4869af4be9eb79247798": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5aee67def05c46d89fe3e97b4f81e3c9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "VBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "VBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "VBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_aa900e7e22674ff2a249eac4bc804a71",
        "IPY_MODEL_81182c1fb849421784f4e8ebb525e3ee"
       ],
       "layout": "IPY_MODEL_af9ddc5fb82f47a7aaa6b0802393026b"
      }
     },
     "70fc441e72a846d69e7e41a24d163063": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "81182c1fb849421784f4e8ebb525e3ee": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_424b1f1a15ad4869af4be9eb79247798",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_24504d5cee27493f879777200b809d3c",
       "value": 0
      }
     },
     "aa900e7e22674ff2a249eac4bc804a71": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "LabelModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "LabelModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "LabelView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_70fc441e72a846d69e7e41a24d163063",
       "placeholder": "​",
       "style": "IPY_MODEL_091c2994d1544a68ab89a94f725c5ed7",
       "value": ""
      }
     },
     "af9ddc5fb82f47a7aaa6b0802393026b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}