CNN_Learning_nets.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CNN Learning nets.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyP3WXwAYBNa3yDWZ9nODHCv"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "0gtYB8P5qkfy"
      },
      "source": [
        "import keras\n",
        "import keras.backend as K\n",
        "from keras.models import Model\n",
        "from keras.layers import Input, Dense, Conv2D, Conv3D, DepthwiseConv2D, SeparableConv2D, Conv3DTranspose\n",
        "from keras.layers import Flatten, MaxPool2D, AvgPool2D, GlobalAvgPool2D, UpSampling2D, BatchNormalization\n",
        "from keras.layers import Concatenate, Add, Dropout, ReLU, Lambda, Activation, LeakyReLU, PReLU\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Flatten\n",
        "from keras.layers import Conv2D, MaxPooling2D\n",
        "from keras import backend as K\n",
        "from IPython.display import SVG\n",
        "from keras.utils.vis_utils import model_to_dot\n",
        "\n",
        "from time import time\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MjvEnw0StZZL",
        "outputId": "fa68bc5a-2a8c-498e-cf50-bfe118d86125",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def squeezenet(input_shape, n_classes):\n",
        "  \n",
        "  def fire(x, fs, fe):\n",
        "    s = Conv2D(fs, 1, activation='relu')(x)\n",
        "    e1 = Conv2D(fe, 1, activation='relu')(s)\n",
        "    e3 = Conv2D(fe, 3, padding='same', activation='relu')(s)\n",
        "    output = Concatenate()([e1, e3])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(96, 7, strides=2, padding='same', activation='relu')(input)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = fire(x, 16, 64)\n",
        "  x = fire(x, 16, 64)\n",
        "  x = fire(x, 32, 128)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = fire(x, 32, 128)\n",
        "  x = fire(x, 48, 192)\n",
        "  x = fire(x, 48, 192)\n",
        "  x = fire(x, 64, 256)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = fire(x, 64, 256)\n",
        "  x = Conv2D(n_classes, 1)(x)\n",
        "  x = GlobalAvgPool2D()(x)\n",
        "  \n",
        "  output = Activation('softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = squeezenet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d (Conv2D)                 (None, 112, 112, 96) 14208       input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D)    (None, 56, 56, 96)   0           conv2d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 56, 56, 16)   1552        max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 56, 56, 64)   1088        conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 56, 56, 64)   9280        conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 56, 56, 128)  0           conv2d_2[0][0]                   \n",
            "                                                                 conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 56, 56, 16)   2064        concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 56, 56, 64)   1088        conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_6 (Conv2D)               (None, 56, 56, 64)   9280        conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 56, 56, 128)  0           conv2d_5[0][0]                   \n",
            "                                                                 conv2d_6[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_7 (Conv2D)               (None, 56, 56, 32)   4128        concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_8 (Conv2D)               (None, 56, 56, 128)  4224        conv2d_7[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 56, 56, 128)  36992       conv2d_7[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_2 (Concatenate)     (None, 56, 56, 256)  0           conv2d_8[0][0]                   \n",
            "                                                                 conv2d_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 28, 28, 256)  0           concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 28, 28, 32)   8224        max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 28, 28, 128)  4224        conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_12 (Conv2D)              (None, 28, 28, 128)  36992       conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)     (None, 28, 28, 256)  0           conv2d_11[0][0]                  \n",
            "                                                                 conv2d_12[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_13 (Conv2D)              (None, 28, 28, 48)   12336       concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_14 (Conv2D)              (None, 28, 28, 192)  9408        conv2d_13[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_15 (Conv2D)              (None, 28, 28, 192)  83136       conv2d_13[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_4 (Concatenate)     (None, 28, 28, 384)  0           conv2d_14[0][0]                  \n",
            "                                                                 conv2d_15[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_16 (Conv2D)              (None, 28, 28, 48)   18480       concatenate_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_17 (Conv2D)              (None, 28, 28, 192)  9408        conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_18 (Conv2D)              (None, 28, 28, 192)  83136       conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_5 (Concatenate)     (None, 28, 28, 384)  0           conv2d_17[0][0]                  \n",
            "                                                                 conv2d_18[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_19 (Conv2D)              (None, 28, 28, 64)   24640       concatenate_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_20 (Conv2D)              (None, 28, 28, 256)  16640       conv2d_19[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_21 (Conv2D)              (None, 28, 28, 256)  147712      conv2d_19[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_6 (Concatenate)     (None, 28, 28, 512)  0           conv2d_20[0][0]                  \n",
            "                                                                 conv2d_21[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2D)  (None, 14, 14, 512)  0           concatenate_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_22 (Conv2D)              (None, 14, 14, 64)   32832       max_pooling2d_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_23 (Conv2D)              (None, 14, 14, 256)  16640       conv2d_22[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_24 (Conv2D)              (None, 14, 14, 256)  147712      conv2d_22[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_7 (Concatenate)     (None, 14, 14, 512)  0           conv2d_23[0][0]                  \n",
            "                                                                 conv2d_24[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_25 (Conv2D)              (None, 14, 14, 1000) 513000      concatenate_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling2d (Globa (None, 1000)         0           conv2d_25[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "activation (Activation)         (None, 1000)         0           global_average_pooling2d[0][0]   \n",
            "==================================================================================================\n",
            "Total params: 1,248,424\n",
            "Trainable params: 1,248,424\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "0.07911496162414551\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OVSmA4c6mtW6",
        "outputId": "3c4a20d3-7115-4d4d-ca22-6ef6aa73e9d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 706
        }
      },
      "source": [
        "def alexnet(input_shape, n_classes):\n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  # actually batch normalization didn't exist back then\n",
        "  # they used LRN (Local Response Normalization) for regularization\n",
        "  x = Conv2D(96, 11, strides=4, padding='same', activation='relu')(input)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = Conv2D(256, 5, padding='same', activation='relu')(x)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = Conv2D(384, 3, strides=1, padding='same', activation='relu')(x)\n",
        "  \n",
        "  x = Conv2D(384, 3, strides=1, padding='same', activation='relu')(x)\n",
        "  \n",
        "  x = Conv2D(256, 3, strides=1, padding='same', activation='relu')(x)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = Flatten()(x)\n",
        "  x = Dense(4096, activation='relu')(x)\n",
        "  x = Dense(4096, activation='relu')(x)\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model\n",
        "\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = alexnet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "conv2d (Conv2D)              (None, 56, 56, 96)        34944     \n",
            "_________________________________________________________________\n",
            "batch_normalization (BatchNo (None, 56, 56, 96)        384       \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 27, 27, 96)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 27, 27, 256)       614656    \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 27, 27, 256)       1024      \n",
            "_________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2 (None, 13, 13, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 13, 13, 384)       885120    \n",
            "_________________________________________________________________\n",
            "conv2d_3 (Conv2D)            (None, 13, 13, 384)       1327488   \n",
            "_________________________________________________________________\n",
            "conv2d_4 (Conv2D)            (None, 13, 13, 256)       884992    \n",
            "_________________________________________________________________\n",
            "batch_normalization_2 (Batch (None, 13, 13, 256)       1024      \n",
            "_________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 256)         0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 9216)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 4096)              37752832  \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 4096)              16781312  \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 1000)              4097000   \n",
            "=================================================================\n",
            "Total params: 62,380,776\n",
            "Trainable params: 62,379,560\n",
            "Non-trainable params: 1,216\n",
            "_________________________________________________________________\n",
            "0.08577654361724854\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BNrd6c2cUnlc",
        "outputId": "fcbbf867-c59b-4a74-a6e9-1a7c2218d663",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def googlenet(input_shape, n_classes):\n",
        "  \n",
        "  def inception_block(x, f):\n",
        "    t1 = Conv2D(f[0], 1, activation='relu')(x)\n",
        "    \n",
        "    t2 = Conv2D(f[1], 1, activation='relu')(x)\n",
        "    t2 = Conv2D(f[2], 3, padding='same', activation='relu')(t2)\n",
        "    \n",
        "    t3 = Conv2D(f[3], 1, activation='relu')(x)\n",
        "    t3 = Conv2D(f[4], 5, padding='same', activation='relu')(t3)\n",
        "    \n",
        "    t4 = MaxPool2D(3, 1, padding='same')(x)\n",
        "    t4 = Conv2D(f[5], 1, activation='relu')(t4)\n",
        "    \n",
        "    output = Concatenate()([t1, t2, t3, t4])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(64, 7, strides=2, padding='same', activation='relu')(input)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = Conv2D(64, 1, activation='relu')(x)\n",
        "  x = Conv2D(192, 3, padding='same', activation='relu')(x)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = inception_block(x, [64, 96, 128, 16, 32, 32])\n",
        "  x = inception_block(x, [128, 128, 192, 32, 96, 64])\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = inception_block(x, [192, 96, 208, 16, 48, 64])\n",
        "  x = inception_block(x, [160, 112, 224, 24, 64, 64])\n",
        "  x = inception_block(x, [128, 128, 256, 24, 64, 64])\n",
        "  x = inception_block(x, [112, 144, 288, 32, 64, 64])\n",
        "  x = inception_block(x, [256, 160, 320, 32, 128, 128])\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "\n",
        "  x = inception_block(x, [256, 160, 320, 32, 128, 128])\n",
        "  x = inception_block(x, [384, 192, 384, 48, 128, 128])\n",
        "  \n",
        "  x = AvgPool2D(7, strides=1)(x)\n",
        "  x = Dropout(0.4)(x)\n",
        "  \n",
        "  x = Flatten()(x)\n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = googlenet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d (Conv2D)                 (None, 112, 112, 64) 9472        input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D)    (None, 56, 56, 64)   0           conv2d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 56, 56, 64)   4160        max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 56, 56, 192)  110784      conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 27, 27, 192)  0           conv2d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 27, 27, 96)   18528       max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_6 (Conv2D)               (None, 27, 27, 16)   3088        max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2D)  (None, 27, 27, 192)  0           max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 27, 27, 64)   12352       max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 27, 27, 128)  110720      conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_7 (Conv2D)               (None, 27, 27, 32)   12832       conv2d_6[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_8 (Conv2D)               (None, 27, 27, 32)   6176        max_pooling2d_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 27, 27, 256)  0           conv2d_3[0][0]                   \n",
            "                                                                 conv2d_5[0][0]                   \n",
            "                                                                 conv2d_7[0][0]                   \n",
            "                                                                 conv2d_8[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 27, 27, 128)  32896       concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_12 (Conv2D)              (None, 27, 27, 32)   8224        concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_3 (MaxPooling2D)  (None, 27, 27, 256)  0           concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 27, 27, 128)  32896       concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 27, 27, 192)  221376      conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_13 (Conv2D)              (None, 27, 27, 96)   76896       conv2d_12[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_14 (Conv2D)              (None, 27, 27, 64)   16448       max_pooling2d_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 27, 27, 480)  0           conv2d_9[0][0]                   \n",
            "                                                                 conv2d_11[0][0]                  \n",
            "                                                                 conv2d_13[0][0]                  \n",
            "                                                                 conv2d_14[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_4 (MaxPooling2D)  (None, 14, 14, 480)  0           concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_16 (Conv2D)              (None, 14, 14, 96)   46176       max_pooling2d_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_18 (Conv2D)              (None, 14, 14, 16)   7696        max_pooling2d_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_5 (MaxPooling2D)  (None, 14, 14, 480)  0           max_pooling2d_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_15 (Conv2D)              (None, 14, 14, 192)  92352       max_pooling2d_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_17 (Conv2D)              (None, 14, 14, 208)  179920      conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_19 (Conv2D)              (None, 14, 14, 48)   19248       conv2d_18[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_20 (Conv2D)              (None, 14, 14, 64)   30784       max_pooling2d_5[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_2 (Concatenate)     (None, 14, 14, 512)  0           conv2d_15[0][0]                  \n",
            "                                                                 conv2d_17[0][0]                  \n",
            "                                                                 conv2d_19[0][0]                  \n",
            "                                                                 conv2d_20[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_22 (Conv2D)              (None, 14, 14, 112)  57456       concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_24 (Conv2D)              (None, 14, 14, 24)   12312       concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_6 (MaxPooling2D)  (None, 14, 14, 512)  0           concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_21 (Conv2D)              (None, 14, 14, 160)  82080       concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_23 (Conv2D)              (None, 14, 14, 224)  226016      conv2d_22[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_25 (Conv2D)              (None, 14, 14, 64)   38464       conv2d_24[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_26 (Conv2D)              (None, 14, 14, 64)   32832       max_pooling2d_6[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)     (None, 14, 14, 512)  0           conv2d_21[0][0]                  \n",
            "                                                                 conv2d_23[0][0]                  \n",
            "                                                                 conv2d_25[0][0]                  \n",
            "                                                                 conv2d_26[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_28 (Conv2D)              (None, 14, 14, 128)  65664       concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_30 (Conv2D)              (None, 14, 14, 24)   12312       concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_7 (MaxPooling2D)  (None, 14, 14, 512)  0           concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_27 (Conv2D)              (None, 14, 14, 128)  65664       concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_29 (Conv2D)              (None, 14, 14, 256)  295168      conv2d_28[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_31 (Conv2D)              (None, 14, 14, 64)   38464       conv2d_30[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_32 (Conv2D)              (None, 14, 14, 64)   32832       max_pooling2d_7[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_4 (Concatenate)     (None, 14, 14, 512)  0           conv2d_27[0][0]                  \n",
            "                                                                 conv2d_29[0][0]                  \n",
            "                                                                 conv2d_31[0][0]                  \n",
            "                                                                 conv2d_32[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_34 (Conv2D)              (None, 14, 14, 144)  73872       concatenate_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_36 (Conv2D)              (None, 14, 14, 32)   16416       concatenate_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_8 (MaxPooling2D)  (None, 14, 14, 512)  0           concatenate_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_33 (Conv2D)              (None, 14, 14, 112)  57456       concatenate_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_35 (Conv2D)              (None, 14, 14, 288)  373536      conv2d_34[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_37 (Conv2D)              (None, 14, 14, 64)   51264       conv2d_36[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_38 (Conv2D)              (None, 14, 14, 64)   32832       max_pooling2d_8[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_5 (Concatenate)     (None, 14, 14, 528)  0           conv2d_33[0][0]                  \n",
            "                                                                 conv2d_35[0][0]                  \n",
            "                                                                 conv2d_37[0][0]                  \n",
            "                                                                 conv2d_38[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_40 (Conv2D)              (None, 14, 14, 160)  84640       concatenate_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_42 (Conv2D)              (None, 14, 14, 32)   16928       concatenate_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_9 (MaxPooling2D)  (None, 14, 14, 528)  0           concatenate_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_39 (Conv2D)              (None, 14, 14, 256)  135424      concatenate_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_41 (Conv2D)              (None, 14, 14, 320)  461120      conv2d_40[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_43 (Conv2D)              (None, 14, 14, 128)  102528      conv2d_42[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_44 (Conv2D)              (None, 14, 14, 128)  67712       max_pooling2d_9[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_6 (Concatenate)     (None, 14, 14, 832)  0           conv2d_39[0][0]                  \n",
            "                                                                 conv2d_41[0][0]                  \n",
            "                                                                 conv2d_43[0][0]                  \n",
            "                                                                 conv2d_44[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_10 (MaxPooling2D) (None, 7, 7, 832)    0           concatenate_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_46 (Conv2D)              (None, 7, 7, 160)    133280      max_pooling2d_10[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_48 (Conv2D)              (None, 7, 7, 32)     26656       max_pooling2d_10[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_11 (MaxPooling2D) (None, 7, 7, 832)    0           max_pooling2d_10[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_45 (Conv2D)              (None, 7, 7, 256)    213248      max_pooling2d_10[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_47 (Conv2D)              (None, 7, 7, 320)    461120      conv2d_46[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_49 (Conv2D)              (None, 7, 7, 128)    102528      conv2d_48[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_50 (Conv2D)              (None, 7, 7, 128)    106624      max_pooling2d_11[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_7 (Concatenate)     (None, 7, 7, 832)    0           conv2d_45[0][0]                  \n",
            "                                                                 conv2d_47[0][0]                  \n",
            "                                                                 conv2d_49[0][0]                  \n",
            "                                                                 conv2d_50[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_52 (Conv2D)              (None, 7, 7, 192)    159936      concatenate_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_54 (Conv2D)              (None, 7, 7, 48)     39984       concatenate_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_12 (MaxPooling2D) (None, 7, 7, 832)    0           concatenate_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_51 (Conv2D)              (None, 7, 7, 384)    319872      concatenate_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_53 (Conv2D)              (None, 7, 7, 384)    663936      conv2d_52[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_55 (Conv2D)              (None, 7, 7, 128)    153728      conv2d_54[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_56 (Conv2D)              (None, 7, 7, 128)    106624      max_pooling2d_12[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_8 (Concatenate)     (None, 7, 7, 1024)   0           conv2d_51[0][0]                  \n",
            "                                                                 conv2d_53[0][0]                  \n",
            "                                                                 conv2d_55[0][0]                  \n",
            "                                                                 conv2d_56[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "average_pooling2d (AveragePooli (None, 1, 1, 1024)   0           concatenate_8[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dropout (Dropout)               (None, 1, 1, 1024)   0           average_pooling2d[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "flatten (Flatten)               (None, 1024)         0           dropout[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 1000)         1025000     flatten[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 6,998,552\n",
            "Trainable params: 6,998,552\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "0.08974912166595458\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3gGNXuqLU99a",
        "outputId": "48fe081c-e181-42fc-fb4b-18b1cbb5f179",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def mobilenet(input_shape, n_classes):\n",
        "  \n",
        "  def mobilenet_block(x, f, s=1):\n",
        "    x = DepthwiseConv2D(3, strides=s, padding='same')(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    \n",
        "    x = Conv2D(f, 1, strides=1, padding='same')(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    return x\n",
        "    \n",
        "    \n",
        "  input = Input(input_shape)\n",
        "\n",
        "  x = Conv2D(32, 3, strides=2, padding='same')(input)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = ReLU()(x)\n",
        "\n",
        "  x = mobilenet_block(x, 64)\n",
        "  x = mobilenet_block(x, 128, 2)\n",
        "  x = mobilenet_block(x, 128)\n",
        "\n",
        "  x = mobilenet_block(x, 256, 2)\n",
        "  x = mobilenet_block(x, 256)\n",
        "\n",
        "  x = mobilenet_block(x, 512, 2)\n",
        "  for _ in range(5):\n",
        "    x = mobilenet_block(x, 512)\n",
        "\n",
        "  x = mobilenet_block(x, 1024, 2)\n",
        "  x = mobilenet_block(x, 1024)\n",
        "  \n",
        "  x = GlobalAvgPool2D()(x)\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model\n",
        "\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = mobilenet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "conv2d (Conv2D)              (None, 112, 112, 32)      896       \n",
            "_________________________________________________________________\n",
            "batch_normalization (BatchNo (None, 112, 112, 32)      128       \n",
            "_________________________________________________________________\n",
            "re_lu (ReLU)                 (None, 112, 112, 32)      0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d (DepthwiseC (None, 112, 112, 32)      320       \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 112, 112, 32)      128       \n",
            "_________________________________________________________________\n",
            "re_lu_1 (ReLU)               (None, 112, 112, 32)      0         \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 112, 112, 64)      2112      \n",
            "_________________________________________________________________\n",
            "batch_normalization_2 (Batch (None, 112, 112, 64)      256       \n",
            "_________________________________________________________________\n",
            "re_lu_2 (ReLU)               (None, 112, 112, 64)      0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_1 (Depthwis (None, 56, 56, 64)        640       \n",
            "_________________________________________________________________\n",
            "batch_normalization_3 (Batch (None, 56, 56, 64)        256       \n",
            "_________________________________________________________________\n",
            "re_lu_3 (ReLU)               (None, 56, 56, 64)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 56, 56, 128)       8320      \n",
            "_________________________________________________________________\n",
            "batch_normalization_4 (Batch (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "re_lu_4 (ReLU)               (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_2 (Depthwis (None, 56, 56, 128)       1280      \n",
            "_________________________________________________________________\n",
            "batch_normalization_5 (Batch (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "re_lu_5 (ReLU)               (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_3 (Conv2D)            (None, 56, 56, 128)       16512     \n",
            "_________________________________________________________________\n",
            "batch_normalization_6 (Batch (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "re_lu_6 (ReLU)               (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_3 (Depthwis (None, 28, 28, 128)       1280      \n",
            "_________________________________________________________________\n",
            "batch_normalization_7 (Batch (None, 28, 28, 128)       512       \n",
            "_________________________________________________________________\n",
            "re_lu_7 (ReLU)               (None, 28, 28, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_4 (Conv2D)            (None, 28, 28, 256)       33024     \n",
            "_________________________________________________________________\n",
            "batch_normalization_8 (Batch (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "re_lu_8 (ReLU)               (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_4 (Depthwis (None, 28, 28, 256)       2560      \n",
            "_________________________________________________________________\n",
            "batch_normalization_9 (Batch (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "re_lu_9 (ReLU)               (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_5 (Conv2D)            (None, 28, 28, 256)       65792     \n",
            "_________________________________________________________________\n",
            "batch_normalization_10 (Batc (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "re_lu_10 (ReLU)              (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_5 (Depthwis (None, 14, 14, 256)       2560      \n",
            "_________________________________________________________________\n",
            "batch_normalization_11 (Batc (None, 14, 14, 256)       1024      \n",
            "_________________________________________________________________\n",
            "re_lu_11 (ReLU)              (None, 14, 14, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_6 (Conv2D)            (None, 14, 14, 512)       131584    \n",
            "_________________________________________________________________\n",
            "batch_normalization_12 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_12 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_6 (Depthwis (None, 14, 14, 512)       5120      \n",
            "_________________________________________________________________\n",
            "batch_normalization_13 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_13 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_7 (Conv2D)            (None, 14, 14, 512)       262656    \n",
            "_________________________________________________________________\n",
            "batch_normalization_14 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_14 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_7 (Depthwis (None, 14, 14, 512)       5120      \n",
            "_________________________________________________________________\n",
            "batch_normalization_15 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_15 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_8 (Conv2D)            (None, 14, 14, 512)       262656    \n",
            "_________________________________________________________________\n",
            "batch_normalization_16 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_16 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_8 (Depthwis (None, 14, 14, 512)       5120      \n",
            "_________________________________________________________________\n",
            "batch_normalization_17 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_17 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_9 (Conv2D)            (None, 14, 14, 512)       262656    \n",
            "_________________________________________________________________\n",
            "batch_normalization_18 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_18 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_9 (Depthwis (None, 14, 14, 512)       5120      \n",
            "_________________________________________________________________\n",
            "batch_normalization_19 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_19 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_10 (Conv2D)           (None, 14, 14, 512)       262656    \n",
            "_________________________________________________________________\n",
            "batch_normalization_20 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_20 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_10 (Depthwi (None, 14, 14, 512)       5120      \n",
            "_________________________________________________________________\n",
            "batch_normalization_21 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_21 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_11 (Conv2D)           (None, 14, 14, 512)       262656    \n",
            "_________________________________________________________________\n",
            "batch_normalization_22 (Batc (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "re_lu_22 (ReLU)              (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_11 (Depthwi (None, 7, 7, 512)         5120      \n",
            "_________________________________________________________________\n",
            "batch_normalization_23 (Batc (None, 7, 7, 512)         2048      \n",
            "_________________________________________________________________\n",
            "re_lu_23 (ReLU)              (None, 7, 7, 512)         0         \n",
            "_________________________________________________________________\n",
            "conv2d_12 (Conv2D)           (None, 7, 7, 1024)        525312    \n",
            "_________________________________________________________________\n",
            "batch_normalization_24 (Batc (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "re_lu_24 (ReLU)              (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "depthwise_conv2d_12 (Depthwi (None, 7, 7, 1024)        10240     \n",
            "_________________________________________________________________\n",
            "batch_normalization_25 (Batc (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "re_lu_25 (ReLU)              (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_13 (Conv2D)           (None, 7, 7, 1024)        1049600   \n",
            "_________________________________________________________________\n",
            "batch_normalization_26 (Batc (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "re_lu_26 (ReLU)              (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "global_average_pooling2d (Gl (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1000)              1025000   \n",
            "=================================================================\n",
            "Total params: 4,264,808\n",
            "Trainable params: 4,242,920\n",
            "Non-trainable params: 21,888\n",
            "_________________________________________________________________\n",
            "0.081136155128479\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7NSMdX_xVRWW",
        "outputId": "88cff8e7-41db-49c9-e828-1b46a4e31b89",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def shufflenet(input_shape, n_classes, g=8):\n",
        "  channels = 384, 769, 1536\n",
        "  repetitions = 3, 7, 3\n",
        "  \n",
        "  def ch_shuffle(x, g):\n",
        "#     1 2 3 4 5 6 7 8 9 -reshape-> 1 2 3 -permute dims-> 1 4 7 -reshape-> 1 4 7 2 5 8 3 6 9\n",
        "#                                  4 5 6                 2 5 8\n",
        "#                                  7 8 9                 3 6 9\n",
        "    \n",
        "    _, w, h, ch = K.int_shape(x)\n",
        "    ch_g = ch // g\n",
        "    \n",
        "    def shuffle_op(x):\n",
        "      x = K.reshape(x, [-1, w, h, ch_g, g])\n",
        "      x = K.permute_dimensions(x, [0, 1, 2, 4, 3])\n",
        "      x = K.reshape(x, [-1, w, h, ch])\n",
        "      return x\n",
        "    \n",
        "    x = Lambda(shuffle_op)(x)\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  def gconv(tensor, ch, g):\n",
        "    _, _, _, in_ch = K.int_shape(tensor)\n",
        "    ch_g = in_ch // g\n",
        "    out_ch = ch // g\n",
        "    group = []\n",
        "    \n",
        "    for i in range(g):\n",
        "#       x = tensor[:, :, :, i*ch_g:(i+1)*ch_g]\n",
        "      x = Lambda(lambda x: x[:, :, :, i*ch_g: (i+1)*ch_g])(tensor)\n",
        "      x = Conv2D(out_ch, 1)(x)\n",
        "      group.append(x)\n",
        "    \n",
        "    x = Concatenate()(group)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def shufflenet_block(tensor, ch, s, g):\n",
        "    x = gconv(tensor, ch // 4, g)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    x = ch_shuffle(x, g)\n",
        "    x = DepthwiseConv2D(3, strides=s, padding='same')(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = gconv(x, ch if s==1 else ch-K.int_shape(tensor)[-1], g)\n",
        "    x = BatchNormalization()(x)\n",
        "    \n",
        "    if s == 1:\n",
        "      x = Add()([tensor, x])\n",
        "    else:\n",
        "      avg = AvgPool2D(3, strides=2, padding='same')(tensor)\n",
        "      x = Concatenate()([avg, x])\n",
        "    \n",
        "    output = ReLU()(x)\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def stage(x, ch, r, g):\n",
        "    x = shufflenet_block(x, ch, 2, g)\n",
        "    \n",
        "    for i in range(r):\n",
        "      x = shufflenet_block(x, ch, 1, g)\n",
        "      \n",
        "    return x\n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(24, 3, strides=2, padding='same')(input)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = ReLU()(x)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  for ch, r in zip(channels, repetitions):\n",
        "    x = stage(x, ch, r, g)\n",
        "    \n",
        "  x = GlobalAvgPool2D()(x)\n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model\n",
        "\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = shufflenet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d (Conv2D)                 (None, 112, 112, 24) 672         input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization (BatchNorma (None, 112, 112, 24) 96          conv2d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu (ReLU)                    (None, 112, 112, 24) 0           batch_normalization[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D)    (None, 56, 56, 24)   0           re_lu[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda (Lambda)                 (None, 56, 56, 3)    0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_1 (Lambda)               (None, 56, 56, 3)    0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_2 (Lambda)               (None, 56, 56, 3)    0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_3 (Lambda)               (None, 56, 56, 3)    0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_4 (Lambda)               (None, 56, 56, 3)    0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_5 (Lambda)               (None, 56, 56, 3)    0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_6 (Lambda)               (None, 56, 56, 3)    0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_7 (Lambda)               (None, 56, 56, 3)    0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 56, 56, 12)   48          lambda[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 56, 56, 12)   48          lambda_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 56, 56, 12)   48          lambda_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 56, 56, 12)   48          lambda_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 56, 56, 12)   48          lambda_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_6 (Conv2D)               (None, 56, 56, 12)   48          lambda_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_7 (Conv2D)               (None, 56, 56, 12)   48          lambda_6[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_8 (Conv2D)               (None, 56, 56, 12)   48          lambda_7[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 56, 56, 96)   0           conv2d_1[0][0]                   \n",
            "                                                                 conv2d_2[0][0]                   \n",
            "                                                                 conv2d_3[0][0]                   \n",
            "                                                                 conv2d_4[0][0]                   \n",
            "                                                                 conv2d_5[0][0]                   \n",
            "                                                                 conv2d_6[0][0]                   \n",
            "                                                                 conv2d_7[0][0]                   \n",
            "                                                                 conv2d_8[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 56, 56, 96)   384         concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_1 (ReLU)                  (None, 56, 56, 96)   0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_8 (Lambda)               (None, 56, 56, 96)   0           re_lu_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d (DepthwiseConv (None, 28, 28, 96)   960         lambda_8[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 28, 28, 96)   384         depthwise_conv2d[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "lambda_9 (Lambda)               (None, 28, 28, 12)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_10 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_11 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_12 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_13 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_14 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_15 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_16 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 28, 28, 45)   585         lambda_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 28, 28, 45)   585         lambda_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 28, 28, 45)   585         lambda_11[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_12 (Conv2D)              (None, 28, 28, 45)   585         lambda_12[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_13 (Conv2D)              (None, 28, 28, 45)   585         lambda_13[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_14 (Conv2D)              (None, 28, 28, 45)   585         lambda_14[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_15 (Conv2D)              (None, 28, 28, 45)   585         lambda_15[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_16 (Conv2D)              (None, 28, 28, 45)   585         lambda_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 28, 28, 360)  0           conv2d_9[0][0]                   \n",
            "                                                                 conv2d_10[0][0]                  \n",
            "                                                                 conv2d_11[0][0]                  \n",
            "                                                                 conv2d_12[0][0]                  \n",
            "                                                                 conv2d_13[0][0]                  \n",
            "                                                                 conv2d_14[0][0]                  \n",
            "                                                                 conv2d_15[0][0]                  \n",
            "                                                                 conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "average_pooling2d (AveragePooli (None, 28, 28, 24)   0           max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 28, 28, 360)  1440        concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_2 (Concatenate)     (None, 28, 28, 384)  0           average_pooling2d[0][0]          \n",
            "                                                                 batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_2 (ReLU)                  (None, 28, 28, 384)  0           concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_17 (Lambda)              (None, 28, 28, 48)   0           re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_18 (Lambda)              (None, 28, 28, 48)   0           re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_19 (Lambda)              (None, 28, 28, 48)   0           re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_20 (Lambda)              (None, 28, 28, 48)   0           re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_21 (Lambda)              (None, 28, 28, 48)   0           re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_22 (Lambda)              (None, 28, 28, 48)   0           re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_23 (Lambda)              (None, 28, 28, 48)   0           re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_24 (Lambda)              (None, 28, 28, 48)   0           re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_17 (Conv2D)              (None, 28, 28, 12)   588         lambda_17[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_18 (Conv2D)              (None, 28, 28, 12)   588         lambda_18[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_19 (Conv2D)              (None, 28, 28, 12)   588         lambda_19[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_20 (Conv2D)              (None, 28, 28, 12)   588         lambda_20[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_21 (Conv2D)              (None, 28, 28, 12)   588         lambda_21[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_22 (Conv2D)              (None, 28, 28, 12)   588         lambda_22[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_23 (Conv2D)              (None, 28, 28, 12)   588         lambda_23[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_24 (Conv2D)              (None, 28, 28, 12)   588         lambda_24[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)     (None, 28, 28, 96)   0           conv2d_17[0][0]                  \n",
            "                                                                 conv2d_18[0][0]                  \n",
            "                                                                 conv2d_19[0][0]                  \n",
            "                                                                 conv2d_20[0][0]                  \n",
            "                                                                 conv2d_21[0][0]                  \n",
            "                                                                 conv2d_22[0][0]                  \n",
            "                                                                 conv2d_23[0][0]                  \n",
            "                                                                 conv2d_24[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 28, 28, 96)   384         concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_3 (ReLU)                  (None, 28, 28, 96)   0           batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_25 (Lambda)              (None, 28, 28, 96)   0           re_lu_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_1 (DepthwiseCo (None, 28, 28, 96)   960         lambda_25[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 28, 28, 96)   384         depthwise_conv2d_1[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_26 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_27 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_28 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_29 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_30 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_31 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_32 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_33 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_25 (Conv2D)              (None, 28, 28, 48)   624         lambda_26[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_26 (Conv2D)              (None, 28, 28, 48)   624         lambda_27[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_27 (Conv2D)              (None, 28, 28, 48)   624         lambda_28[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_28 (Conv2D)              (None, 28, 28, 48)   624         lambda_29[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_29 (Conv2D)              (None, 28, 28, 48)   624         lambda_30[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_30 (Conv2D)              (None, 28, 28, 48)   624         lambda_31[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_31 (Conv2D)              (None, 28, 28, 48)   624         lambda_32[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_32 (Conv2D)              (None, 28, 28, 48)   624         lambda_33[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_4 (Concatenate)     (None, 28, 28, 384)  0           conv2d_25[0][0]                  \n",
            "                                                                 conv2d_26[0][0]                  \n",
            "                                                                 conv2d_27[0][0]                  \n",
            "                                                                 conv2d_28[0][0]                  \n",
            "                                                                 conv2d_29[0][0]                  \n",
            "                                                                 conv2d_30[0][0]                  \n",
            "                                                                 conv2d_31[0][0]                  \n",
            "                                                                 conv2d_32[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 28, 28, 384)  1536        concatenate_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "add (Add)                       (None, 28, 28, 384)  0           re_lu_2[0][0]                    \n",
            "                                                                 batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_4 (ReLU)                  (None, 28, 28, 384)  0           add[0][0]                        \n",
            "__________________________________________________________________________________________________\n",
            "lambda_34 (Lambda)              (None, 28, 28, 48)   0           re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_35 (Lambda)              (None, 28, 28, 48)   0           re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_36 (Lambda)              (None, 28, 28, 48)   0           re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_37 (Lambda)              (None, 28, 28, 48)   0           re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_38 (Lambda)              (None, 28, 28, 48)   0           re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_39 (Lambda)              (None, 28, 28, 48)   0           re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_40 (Lambda)              (None, 28, 28, 48)   0           re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_41 (Lambda)              (None, 28, 28, 48)   0           re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_33 (Conv2D)              (None, 28, 28, 12)   588         lambda_34[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_34 (Conv2D)              (None, 28, 28, 12)   588         lambda_35[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_35 (Conv2D)              (None, 28, 28, 12)   588         lambda_36[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_36 (Conv2D)              (None, 28, 28, 12)   588         lambda_37[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_37 (Conv2D)              (None, 28, 28, 12)   588         lambda_38[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_38 (Conv2D)              (None, 28, 28, 12)   588         lambda_39[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_39 (Conv2D)              (None, 28, 28, 12)   588         lambda_40[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_40 (Conv2D)              (None, 28, 28, 12)   588         lambda_41[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_5 (Concatenate)     (None, 28, 28, 96)   0           conv2d_33[0][0]                  \n",
            "                                                                 conv2d_34[0][0]                  \n",
            "                                                                 conv2d_35[0][0]                  \n",
            "                                                                 conv2d_36[0][0]                  \n",
            "                                                                 conv2d_37[0][0]                  \n",
            "                                                                 conv2d_38[0][0]                  \n",
            "                                                                 conv2d_39[0][0]                  \n",
            "                                                                 conv2d_40[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_7 (BatchNor (None, 28, 28, 96)   384         concatenate_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_5 (ReLU)                  (None, 28, 28, 96)   0           batch_normalization_7[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_42 (Lambda)              (None, 28, 28, 96)   0           re_lu_5[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_2 (DepthwiseCo (None, 28, 28, 96)   960         lambda_42[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_8 (BatchNor (None, 28, 28, 96)   384         depthwise_conv2d_2[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_43 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_44 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_45 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_46 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_47 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_48 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_49 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_50 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_41 (Conv2D)              (None, 28, 28, 48)   624         lambda_43[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_42 (Conv2D)              (None, 28, 28, 48)   624         lambda_44[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_43 (Conv2D)              (None, 28, 28, 48)   624         lambda_45[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_44 (Conv2D)              (None, 28, 28, 48)   624         lambda_46[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_45 (Conv2D)              (None, 28, 28, 48)   624         lambda_47[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_46 (Conv2D)              (None, 28, 28, 48)   624         lambda_48[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_47 (Conv2D)              (None, 28, 28, 48)   624         lambda_49[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_48 (Conv2D)              (None, 28, 28, 48)   624         lambda_50[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_6 (Concatenate)     (None, 28, 28, 384)  0           conv2d_41[0][0]                  \n",
            "                                                                 conv2d_42[0][0]                  \n",
            "                                                                 conv2d_43[0][0]                  \n",
            "                                                                 conv2d_44[0][0]                  \n",
            "                                                                 conv2d_45[0][0]                  \n",
            "                                                                 conv2d_46[0][0]                  \n",
            "                                                                 conv2d_47[0][0]                  \n",
            "                                                                 conv2d_48[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_9 (BatchNor (None, 28, 28, 384)  1536        concatenate_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "add_1 (Add)                     (None, 28, 28, 384)  0           re_lu_4[0][0]                    \n",
            "                                                                 batch_normalization_9[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_6 (ReLU)                  (None, 28, 28, 384)  0           add_1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_51 (Lambda)              (None, 28, 28, 48)   0           re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_52 (Lambda)              (None, 28, 28, 48)   0           re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_53 (Lambda)              (None, 28, 28, 48)   0           re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_54 (Lambda)              (None, 28, 28, 48)   0           re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_55 (Lambda)              (None, 28, 28, 48)   0           re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_56 (Lambda)              (None, 28, 28, 48)   0           re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_57 (Lambda)              (None, 28, 28, 48)   0           re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_58 (Lambda)              (None, 28, 28, 48)   0           re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_49 (Conv2D)              (None, 28, 28, 12)   588         lambda_51[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_50 (Conv2D)              (None, 28, 28, 12)   588         lambda_52[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_51 (Conv2D)              (None, 28, 28, 12)   588         lambda_53[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_52 (Conv2D)              (None, 28, 28, 12)   588         lambda_54[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_53 (Conv2D)              (None, 28, 28, 12)   588         lambda_55[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_54 (Conv2D)              (None, 28, 28, 12)   588         lambda_56[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_55 (Conv2D)              (None, 28, 28, 12)   588         lambda_57[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_56 (Conv2D)              (None, 28, 28, 12)   588         lambda_58[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_7 (Concatenate)     (None, 28, 28, 96)   0           conv2d_49[0][0]                  \n",
            "                                                                 conv2d_50[0][0]                  \n",
            "                                                                 conv2d_51[0][0]                  \n",
            "                                                                 conv2d_52[0][0]                  \n",
            "                                                                 conv2d_53[0][0]                  \n",
            "                                                                 conv2d_54[0][0]                  \n",
            "                                                                 conv2d_55[0][0]                  \n",
            "                                                                 conv2d_56[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_10 (BatchNo (None, 28, 28, 96)   384         concatenate_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_7 (ReLU)                  (None, 28, 28, 96)   0           batch_normalization_10[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_59 (Lambda)              (None, 28, 28, 96)   0           re_lu_7[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_3 (DepthwiseCo (None, 28, 28, 96)   960         lambda_59[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_11 (BatchNo (None, 28, 28, 96)   384         depthwise_conv2d_3[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_60 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_61 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_62 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_63 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_64 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_65 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_66 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_67 (Lambda)              (None, 28, 28, 12)   0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_57 (Conv2D)              (None, 28, 28, 48)   624         lambda_60[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_58 (Conv2D)              (None, 28, 28, 48)   624         lambda_61[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_59 (Conv2D)              (None, 28, 28, 48)   624         lambda_62[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_60 (Conv2D)              (None, 28, 28, 48)   624         lambda_63[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_61 (Conv2D)              (None, 28, 28, 48)   624         lambda_64[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_62 (Conv2D)              (None, 28, 28, 48)   624         lambda_65[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_63 (Conv2D)              (None, 28, 28, 48)   624         lambda_66[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_64 (Conv2D)              (None, 28, 28, 48)   624         lambda_67[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_8 (Concatenate)     (None, 28, 28, 384)  0           conv2d_57[0][0]                  \n",
            "                                                                 conv2d_58[0][0]                  \n",
            "                                                                 conv2d_59[0][0]                  \n",
            "                                                                 conv2d_60[0][0]                  \n",
            "                                                                 conv2d_61[0][0]                  \n",
            "                                                                 conv2d_62[0][0]                  \n",
            "                                                                 conv2d_63[0][0]                  \n",
            "                                                                 conv2d_64[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_12 (BatchNo (None, 28, 28, 384)  1536        concatenate_8[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "add_2 (Add)                     (None, 28, 28, 384)  0           re_lu_6[0][0]                    \n",
            "                                                                 batch_normalization_12[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_8 (ReLU)                  (None, 28, 28, 384)  0           add_2[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_68 (Lambda)              (None, 28, 28, 48)   0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_69 (Lambda)              (None, 28, 28, 48)   0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_70 (Lambda)              (None, 28, 28, 48)   0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_71 (Lambda)              (None, 28, 28, 48)   0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_72 (Lambda)              (None, 28, 28, 48)   0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_73 (Lambda)              (None, 28, 28, 48)   0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_74 (Lambda)              (None, 28, 28, 48)   0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_75 (Lambda)              (None, 28, 28, 48)   0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_65 (Conv2D)              (None, 28, 28, 24)   1176        lambda_68[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_66 (Conv2D)              (None, 28, 28, 24)   1176        lambda_69[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_67 (Conv2D)              (None, 28, 28, 24)   1176        lambda_70[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_68 (Conv2D)              (None, 28, 28, 24)   1176        lambda_71[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_69 (Conv2D)              (None, 28, 28, 24)   1176        lambda_72[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_70 (Conv2D)              (None, 28, 28, 24)   1176        lambda_73[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_71 (Conv2D)              (None, 28, 28, 24)   1176        lambda_74[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_72 (Conv2D)              (None, 28, 28, 24)   1176        lambda_75[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_9 (Concatenate)     (None, 28, 28, 192)  0           conv2d_65[0][0]                  \n",
            "                                                                 conv2d_66[0][0]                  \n",
            "                                                                 conv2d_67[0][0]                  \n",
            "                                                                 conv2d_68[0][0]                  \n",
            "                                                                 conv2d_69[0][0]                  \n",
            "                                                                 conv2d_70[0][0]                  \n",
            "                                                                 conv2d_71[0][0]                  \n",
            "                                                                 conv2d_72[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_13 (BatchNo (None, 28, 28, 192)  768         concatenate_9[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_9 (ReLU)                  (None, 28, 28, 192)  0           batch_normalization_13[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_76 (Lambda)              (None, 28, 28, 192)  0           re_lu_9[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_4 (DepthwiseCo (None, 14, 14, 192)  1920        lambda_76[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_14 (BatchNo (None, 14, 14, 192)  768         depthwise_conv2d_4[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_77 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_78 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_79 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_80 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_81 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_82 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_83 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_84 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_73 (Conv2D)              (None, 14, 14, 48)   1200        lambda_77[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_74 (Conv2D)              (None, 14, 14, 48)   1200        lambda_78[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_75 (Conv2D)              (None, 14, 14, 48)   1200        lambda_79[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_76 (Conv2D)              (None, 14, 14, 48)   1200        lambda_80[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_77 (Conv2D)              (None, 14, 14, 48)   1200        lambda_81[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_78 (Conv2D)              (None, 14, 14, 48)   1200        lambda_82[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_79 (Conv2D)              (None, 14, 14, 48)   1200        lambda_83[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_80 (Conv2D)              (None, 14, 14, 48)   1200        lambda_84[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_10 (Concatenate)    (None, 14, 14, 384)  0           conv2d_73[0][0]                  \n",
            "                                                                 conv2d_74[0][0]                  \n",
            "                                                                 conv2d_75[0][0]                  \n",
            "                                                                 conv2d_76[0][0]                  \n",
            "                                                                 conv2d_77[0][0]                  \n",
            "                                                                 conv2d_78[0][0]                  \n",
            "                                                                 conv2d_79[0][0]                  \n",
            "                                                                 conv2d_80[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "average_pooling2d_1 (AveragePoo (None, 14, 14, 384)  0           re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_15 (BatchNo (None, 14, 14, 384)  1536        concatenate_10[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_11 (Concatenate)    (None, 14, 14, 768)  0           average_pooling2d_1[0][0]        \n",
            "                                                                 batch_normalization_15[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_10 (ReLU)                 (None, 14, 14, 768)  0           concatenate_11[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "lambda_85 (Lambda)              (None, 14, 14, 96)   0           re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_86 (Lambda)              (None, 14, 14, 96)   0           re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_87 (Lambda)              (None, 14, 14, 96)   0           re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_88 (Lambda)              (None, 14, 14, 96)   0           re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_89 (Lambda)              (None, 14, 14, 96)   0           re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_90 (Lambda)              (None, 14, 14, 96)   0           re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_91 (Lambda)              (None, 14, 14, 96)   0           re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_92 (Lambda)              (None, 14, 14, 96)   0           re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_81 (Conv2D)              (None, 14, 14, 24)   2328        lambda_85[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_82 (Conv2D)              (None, 14, 14, 24)   2328        lambda_86[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_83 (Conv2D)              (None, 14, 14, 24)   2328        lambda_87[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_84 (Conv2D)              (None, 14, 14, 24)   2328        lambda_88[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_85 (Conv2D)              (None, 14, 14, 24)   2328        lambda_89[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_86 (Conv2D)              (None, 14, 14, 24)   2328        lambda_90[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_87 (Conv2D)              (None, 14, 14, 24)   2328        lambda_91[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_88 (Conv2D)              (None, 14, 14, 24)   2328        lambda_92[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_12 (Concatenate)    (None, 14, 14, 192)  0           conv2d_81[0][0]                  \n",
            "                                                                 conv2d_82[0][0]                  \n",
            "                                                                 conv2d_83[0][0]                  \n",
            "                                                                 conv2d_84[0][0]                  \n",
            "                                                                 conv2d_85[0][0]                  \n",
            "                                                                 conv2d_86[0][0]                  \n",
            "                                                                 conv2d_87[0][0]                  \n",
            "                                                                 conv2d_88[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_16 (BatchNo (None, 14, 14, 192)  768         concatenate_12[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_11 (ReLU)                 (None, 14, 14, 192)  0           batch_normalization_16[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_93 (Lambda)              (None, 14, 14, 192)  0           re_lu_11[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_5 (DepthwiseCo (None, 14, 14, 192)  1920        lambda_93[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_17 (BatchNo (None, 14, 14, 192)  768         depthwise_conv2d_5[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_94 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_95 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_96 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_97 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_98 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_99 (Lambda)              (None, 14, 14, 24)   0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_100 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_101 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_89 (Conv2D)              (None, 14, 14, 96)   2400        lambda_94[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_90 (Conv2D)              (None, 14, 14, 96)   2400        lambda_95[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_91 (Conv2D)              (None, 14, 14, 96)   2400        lambda_96[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_92 (Conv2D)              (None, 14, 14, 96)   2400        lambda_97[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_93 (Conv2D)              (None, 14, 14, 96)   2400        lambda_98[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_94 (Conv2D)              (None, 14, 14, 96)   2400        lambda_99[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_95 (Conv2D)              (None, 14, 14, 96)   2400        lambda_100[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_96 (Conv2D)              (None, 14, 14, 96)   2400        lambda_101[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_13 (Concatenate)    (None, 14, 14, 768)  0           conv2d_89[0][0]                  \n",
            "                                                                 conv2d_90[0][0]                  \n",
            "                                                                 conv2d_91[0][0]                  \n",
            "                                                                 conv2d_92[0][0]                  \n",
            "                                                                 conv2d_93[0][0]                  \n",
            "                                                                 conv2d_94[0][0]                  \n",
            "                                                                 conv2d_95[0][0]                  \n",
            "                                                                 conv2d_96[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_18 (BatchNo (None, 14, 14, 768)  3072        concatenate_13[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_3 (Add)                     (None, 14, 14, 768)  0           re_lu_10[0][0]                   \n",
            "                                                                 batch_normalization_18[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_12 (ReLU)                 (None, 14, 14, 768)  0           add_3[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_102 (Lambda)             (None, 14, 14, 96)   0           re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_103 (Lambda)             (None, 14, 14, 96)   0           re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_104 (Lambda)             (None, 14, 14, 96)   0           re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_105 (Lambda)             (None, 14, 14, 96)   0           re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_106 (Lambda)             (None, 14, 14, 96)   0           re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_107 (Lambda)             (None, 14, 14, 96)   0           re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_108 (Lambda)             (None, 14, 14, 96)   0           re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_109 (Lambda)             (None, 14, 14, 96)   0           re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_97 (Conv2D)              (None, 14, 14, 24)   2328        lambda_102[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_98 (Conv2D)              (None, 14, 14, 24)   2328        lambda_103[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_99 (Conv2D)              (None, 14, 14, 24)   2328        lambda_104[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_100 (Conv2D)             (None, 14, 14, 24)   2328        lambda_105[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_101 (Conv2D)             (None, 14, 14, 24)   2328        lambda_106[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_102 (Conv2D)             (None, 14, 14, 24)   2328        lambda_107[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_103 (Conv2D)             (None, 14, 14, 24)   2328        lambda_108[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_104 (Conv2D)             (None, 14, 14, 24)   2328        lambda_109[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_14 (Concatenate)    (None, 14, 14, 192)  0           conv2d_97[0][0]                  \n",
            "                                                                 conv2d_98[0][0]                  \n",
            "                                                                 conv2d_99[0][0]                  \n",
            "                                                                 conv2d_100[0][0]                 \n",
            "                                                                 conv2d_101[0][0]                 \n",
            "                                                                 conv2d_102[0][0]                 \n",
            "                                                                 conv2d_103[0][0]                 \n",
            "                                                                 conv2d_104[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_19 (BatchNo (None, 14, 14, 192)  768         concatenate_14[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_13 (ReLU)                 (None, 14, 14, 192)  0           batch_normalization_19[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_110 (Lambda)             (None, 14, 14, 192)  0           re_lu_13[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_6 (DepthwiseCo (None, 14, 14, 192)  1920        lambda_110[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_20 (BatchNo (None, 14, 14, 192)  768         depthwise_conv2d_6[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_111 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_112 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_113 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_114 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_115 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_116 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_117 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_118 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_105 (Conv2D)             (None, 14, 14, 96)   2400        lambda_111[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_106 (Conv2D)             (None, 14, 14, 96)   2400        lambda_112[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_107 (Conv2D)             (None, 14, 14, 96)   2400        lambda_113[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_108 (Conv2D)             (None, 14, 14, 96)   2400        lambda_114[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_109 (Conv2D)             (None, 14, 14, 96)   2400        lambda_115[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_110 (Conv2D)             (None, 14, 14, 96)   2400        lambda_116[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_111 (Conv2D)             (None, 14, 14, 96)   2400        lambda_117[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_112 (Conv2D)             (None, 14, 14, 96)   2400        lambda_118[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_15 (Concatenate)    (None, 14, 14, 768)  0           conv2d_105[0][0]                 \n",
            "                                                                 conv2d_106[0][0]                 \n",
            "                                                                 conv2d_107[0][0]                 \n",
            "                                                                 conv2d_108[0][0]                 \n",
            "                                                                 conv2d_109[0][0]                 \n",
            "                                                                 conv2d_110[0][0]                 \n",
            "                                                                 conv2d_111[0][0]                 \n",
            "                                                                 conv2d_112[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_21 (BatchNo (None, 14, 14, 768)  3072        concatenate_15[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_4 (Add)                     (None, 14, 14, 768)  0           re_lu_12[0][0]                   \n",
            "                                                                 batch_normalization_21[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_14 (ReLU)                 (None, 14, 14, 768)  0           add_4[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_119 (Lambda)             (None, 14, 14, 96)   0           re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_120 (Lambda)             (None, 14, 14, 96)   0           re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_121 (Lambda)             (None, 14, 14, 96)   0           re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_122 (Lambda)             (None, 14, 14, 96)   0           re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_123 (Lambda)             (None, 14, 14, 96)   0           re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_124 (Lambda)             (None, 14, 14, 96)   0           re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_125 (Lambda)             (None, 14, 14, 96)   0           re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_126 (Lambda)             (None, 14, 14, 96)   0           re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_113 (Conv2D)             (None, 14, 14, 24)   2328        lambda_119[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_114 (Conv2D)             (None, 14, 14, 24)   2328        lambda_120[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_115 (Conv2D)             (None, 14, 14, 24)   2328        lambda_121[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_116 (Conv2D)             (None, 14, 14, 24)   2328        lambda_122[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_117 (Conv2D)             (None, 14, 14, 24)   2328        lambda_123[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_118 (Conv2D)             (None, 14, 14, 24)   2328        lambda_124[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_119 (Conv2D)             (None, 14, 14, 24)   2328        lambda_125[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_120 (Conv2D)             (None, 14, 14, 24)   2328        lambda_126[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_16 (Concatenate)    (None, 14, 14, 192)  0           conv2d_113[0][0]                 \n",
            "                                                                 conv2d_114[0][0]                 \n",
            "                                                                 conv2d_115[0][0]                 \n",
            "                                                                 conv2d_116[0][0]                 \n",
            "                                                                 conv2d_117[0][0]                 \n",
            "                                                                 conv2d_118[0][0]                 \n",
            "                                                                 conv2d_119[0][0]                 \n",
            "                                                                 conv2d_120[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_22 (BatchNo (None, 14, 14, 192)  768         concatenate_16[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_15 (ReLU)                 (None, 14, 14, 192)  0           batch_normalization_22[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_127 (Lambda)             (None, 14, 14, 192)  0           re_lu_15[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_7 (DepthwiseCo (None, 14, 14, 192)  1920        lambda_127[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_23 (BatchNo (None, 14, 14, 192)  768         depthwise_conv2d_7[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_128 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_129 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_130 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_131 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_132 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_133 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_134 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_135 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_121 (Conv2D)             (None, 14, 14, 96)   2400        lambda_128[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_122 (Conv2D)             (None, 14, 14, 96)   2400        lambda_129[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_123 (Conv2D)             (None, 14, 14, 96)   2400        lambda_130[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_124 (Conv2D)             (None, 14, 14, 96)   2400        lambda_131[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_125 (Conv2D)             (None, 14, 14, 96)   2400        lambda_132[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_126 (Conv2D)             (None, 14, 14, 96)   2400        lambda_133[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_127 (Conv2D)             (None, 14, 14, 96)   2400        lambda_134[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_128 (Conv2D)             (None, 14, 14, 96)   2400        lambda_135[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_17 (Concatenate)    (None, 14, 14, 768)  0           conv2d_121[0][0]                 \n",
            "                                                                 conv2d_122[0][0]                 \n",
            "                                                                 conv2d_123[0][0]                 \n",
            "                                                                 conv2d_124[0][0]                 \n",
            "                                                                 conv2d_125[0][0]                 \n",
            "                                                                 conv2d_126[0][0]                 \n",
            "                                                                 conv2d_127[0][0]                 \n",
            "                                                                 conv2d_128[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_24 (BatchNo (None, 14, 14, 768)  3072        concatenate_17[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_5 (Add)                     (None, 14, 14, 768)  0           re_lu_14[0][0]                   \n",
            "                                                                 batch_normalization_24[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_16 (ReLU)                 (None, 14, 14, 768)  0           add_5[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_136 (Lambda)             (None, 14, 14, 96)   0           re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_137 (Lambda)             (None, 14, 14, 96)   0           re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_138 (Lambda)             (None, 14, 14, 96)   0           re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_139 (Lambda)             (None, 14, 14, 96)   0           re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_140 (Lambda)             (None, 14, 14, 96)   0           re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_141 (Lambda)             (None, 14, 14, 96)   0           re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_142 (Lambda)             (None, 14, 14, 96)   0           re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_143 (Lambda)             (None, 14, 14, 96)   0           re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_129 (Conv2D)             (None, 14, 14, 24)   2328        lambda_136[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_130 (Conv2D)             (None, 14, 14, 24)   2328        lambda_137[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_131 (Conv2D)             (None, 14, 14, 24)   2328        lambda_138[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_132 (Conv2D)             (None, 14, 14, 24)   2328        lambda_139[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_133 (Conv2D)             (None, 14, 14, 24)   2328        lambda_140[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_134 (Conv2D)             (None, 14, 14, 24)   2328        lambda_141[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_135 (Conv2D)             (None, 14, 14, 24)   2328        lambda_142[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_136 (Conv2D)             (None, 14, 14, 24)   2328        lambda_143[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_18 (Concatenate)    (None, 14, 14, 192)  0           conv2d_129[0][0]                 \n",
            "                                                                 conv2d_130[0][0]                 \n",
            "                                                                 conv2d_131[0][0]                 \n",
            "                                                                 conv2d_132[0][0]                 \n",
            "                                                                 conv2d_133[0][0]                 \n",
            "                                                                 conv2d_134[0][0]                 \n",
            "                                                                 conv2d_135[0][0]                 \n",
            "                                                                 conv2d_136[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_25 (BatchNo (None, 14, 14, 192)  768         concatenate_18[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_17 (ReLU)                 (None, 14, 14, 192)  0           batch_normalization_25[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_144 (Lambda)             (None, 14, 14, 192)  0           re_lu_17[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_8 (DepthwiseCo (None, 14, 14, 192)  1920        lambda_144[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_26 (BatchNo (None, 14, 14, 192)  768         depthwise_conv2d_8[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_145 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_146 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_147 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_148 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_149 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_150 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_151 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_152 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_137 (Conv2D)             (None, 14, 14, 96)   2400        lambda_145[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_138 (Conv2D)             (None, 14, 14, 96)   2400        lambda_146[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_139 (Conv2D)             (None, 14, 14, 96)   2400        lambda_147[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_140 (Conv2D)             (None, 14, 14, 96)   2400        lambda_148[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_141 (Conv2D)             (None, 14, 14, 96)   2400        lambda_149[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_142 (Conv2D)             (None, 14, 14, 96)   2400        lambda_150[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_143 (Conv2D)             (None, 14, 14, 96)   2400        lambda_151[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_144 (Conv2D)             (None, 14, 14, 96)   2400        lambda_152[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_19 (Concatenate)    (None, 14, 14, 768)  0           conv2d_137[0][0]                 \n",
            "                                                                 conv2d_138[0][0]                 \n",
            "                                                                 conv2d_139[0][0]                 \n",
            "                                                                 conv2d_140[0][0]                 \n",
            "                                                                 conv2d_141[0][0]                 \n",
            "                                                                 conv2d_142[0][0]                 \n",
            "                                                                 conv2d_143[0][0]                 \n",
            "                                                                 conv2d_144[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_27 (BatchNo (None, 14, 14, 768)  3072        concatenate_19[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_6 (Add)                     (None, 14, 14, 768)  0           re_lu_16[0][0]                   \n",
            "                                                                 batch_normalization_27[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_18 (ReLU)                 (None, 14, 14, 768)  0           add_6[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_153 (Lambda)             (None, 14, 14, 96)   0           re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_154 (Lambda)             (None, 14, 14, 96)   0           re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_155 (Lambda)             (None, 14, 14, 96)   0           re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_156 (Lambda)             (None, 14, 14, 96)   0           re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_157 (Lambda)             (None, 14, 14, 96)   0           re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_158 (Lambda)             (None, 14, 14, 96)   0           re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_159 (Lambda)             (None, 14, 14, 96)   0           re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_160 (Lambda)             (None, 14, 14, 96)   0           re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_145 (Conv2D)             (None, 14, 14, 24)   2328        lambda_153[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_146 (Conv2D)             (None, 14, 14, 24)   2328        lambda_154[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_147 (Conv2D)             (None, 14, 14, 24)   2328        lambda_155[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_148 (Conv2D)             (None, 14, 14, 24)   2328        lambda_156[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_149 (Conv2D)             (None, 14, 14, 24)   2328        lambda_157[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_150 (Conv2D)             (None, 14, 14, 24)   2328        lambda_158[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_151 (Conv2D)             (None, 14, 14, 24)   2328        lambda_159[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_152 (Conv2D)             (None, 14, 14, 24)   2328        lambda_160[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_20 (Concatenate)    (None, 14, 14, 192)  0           conv2d_145[0][0]                 \n",
            "                                                                 conv2d_146[0][0]                 \n",
            "                                                                 conv2d_147[0][0]                 \n",
            "                                                                 conv2d_148[0][0]                 \n",
            "                                                                 conv2d_149[0][0]                 \n",
            "                                                                 conv2d_150[0][0]                 \n",
            "                                                                 conv2d_151[0][0]                 \n",
            "                                                                 conv2d_152[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_28 (BatchNo (None, 14, 14, 192)  768         concatenate_20[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_19 (ReLU)                 (None, 14, 14, 192)  0           batch_normalization_28[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_161 (Lambda)             (None, 14, 14, 192)  0           re_lu_19[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_9 (DepthwiseCo (None, 14, 14, 192)  1920        lambda_161[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_29 (BatchNo (None, 14, 14, 192)  768         depthwise_conv2d_9[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "lambda_162 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_163 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_164 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_165 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_166 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_167 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_168 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_169 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_153 (Conv2D)             (None, 14, 14, 96)   2400        lambda_162[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_154 (Conv2D)             (None, 14, 14, 96)   2400        lambda_163[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_155 (Conv2D)             (None, 14, 14, 96)   2400        lambda_164[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_156 (Conv2D)             (None, 14, 14, 96)   2400        lambda_165[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_157 (Conv2D)             (None, 14, 14, 96)   2400        lambda_166[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_158 (Conv2D)             (None, 14, 14, 96)   2400        lambda_167[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_159 (Conv2D)             (None, 14, 14, 96)   2400        lambda_168[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_160 (Conv2D)             (None, 14, 14, 96)   2400        lambda_169[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_21 (Concatenate)    (None, 14, 14, 768)  0           conv2d_153[0][0]                 \n",
            "                                                                 conv2d_154[0][0]                 \n",
            "                                                                 conv2d_155[0][0]                 \n",
            "                                                                 conv2d_156[0][0]                 \n",
            "                                                                 conv2d_157[0][0]                 \n",
            "                                                                 conv2d_158[0][0]                 \n",
            "                                                                 conv2d_159[0][0]                 \n",
            "                                                                 conv2d_160[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_30 (BatchNo (None, 14, 14, 768)  3072        concatenate_21[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_7 (Add)                     (None, 14, 14, 768)  0           re_lu_18[0][0]                   \n",
            "                                                                 batch_normalization_30[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_20 (ReLU)                 (None, 14, 14, 768)  0           add_7[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_170 (Lambda)             (None, 14, 14, 96)   0           re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_171 (Lambda)             (None, 14, 14, 96)   0           re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_172 (Lambda)             (None, 14, 14, 96)   0           re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_173 (Lambda)             (None, 14, 14, 96)   0           re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_174 (Lambda)             (None, 14, 14, 96)   0           re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_175 (Lambda)             (None, 14, 14, 96)   0           re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_176 (Lambda)             (None, 14, 14, 96)   0           re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_177 (Lambda)             (None, 14, 14, 96)   0           re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_161 (Conv2D)             (None, 14, 14, 24)   2328        lambda_170[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_162 (Conv2D)             (None, 14, 14, 24)   2328        lambda_171[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_163 (Conv2D)             (None, 14, 14, 24)   2328        lambda_172[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_164 (Conv2D)             (None, 14, 14, 24)   2328        lambda_173[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_165 (Conv2D)             (None, 14, 14, 24)   2328        lambda_174[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_166 (Conv2D)             (None, 14, 14, 24)   2328        lambda_175[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_167 (Conv2D)             (None, 14, 14, 24)   2328        lambda_176[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_168 (Conv2D)             (None, 14, 14, 24)   2328        lambda_177[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_22 (Concatenate)    (None, 14, 14, 192)  0           conv2d_161[0][0]                 \n",
            "                                                                 conv2d_162[0][0]                 \n",
            "                                                                 conv2d_163[0][0]                 \n",
            "                                                                 conv2d_164[0][0]                 \n",
            "                                                                 conv2d_165[0][0]                 \n",
            "                                                                 conv2d_166[0][0]                 \n",
            "                                                                 conv2d_167[0][0]                 \n",
            "                                                                 conv2d_168[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_31 (BatchNo (None, 14, 14, 192)  768         concatenate_22[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_21 (ReLU)                 (None, 14, 14, 192)  0           batch_normalization_31[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_178 (Lambda)             (None, 14, 14, 192)  0           re_lu_21[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_10 (DepthwiseC (None, 14, 14, 192)  1920        lambda_178[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_32 (BatchNo (None, 14, 14, 192)  768         depthwise_conv2d_10[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "lambda_179 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_180 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_181 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_182 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_183 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_184 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_185 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_186 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_169 (Conv2D)             (None, 14, 14, 96)   2400        lambda_179[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_170 (Conv2D)             (None, 14, 14, 96)   2400        lambda_180[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_171 (Conv2D)             (None, 14, 14, 96)   2400        lambda_181[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_172 (Conv2D)             (None, 14, 14, 96)   2400        lambda_182[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_173 (Conv2D)             (None, 14, 14, 96)   2400        lambda_183[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_174 (Conv2D)             (None, 14, 14, 96)   2400        lambda_184[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_175 (Conv2D)             (None, 14, 14, 96)   2400        lambda_185[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_176 (Conv2D)             (None, 14, 14, 96)   2400        lambda_186[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_23 (Concatenate)    (None, 14, 14, 768)  0           conv2d_169[0][0]                 \n",
            "                                                                 conv2d_170[0][0]                 \n",
            "                                                                 conv2d_171[0][0]                 \n",
            "                                                                 conv2d_172[0][0]                 \n",
            "                                                                 conv2d_173[0][0]                 \n",
            "                                                                 conv2d_174[0][0]                 \n",
            "                                                                 conv2d_175[0][0]                 \n",
            "                                                                 conv2d_176[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_33 (BatchNo (None, 14, 14, 768)  3072        concatenate_23[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_8 (Add)                     (None, 14, 14, 768)  0           re_lu_20[0][0]                   \n",
            "                                                                 batch_normalization_33[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_22 (ReLU)                 (None, 14, 14, 768)  0           add_8[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_187 (Lambda)             (None, 14, 14, 96)   0           re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_188 (Lambda)             (None, 14, 14, 96)   0           re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_189 (Lambda)             (None, 14, 14, 96)   0           re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_190 (Lambda)             (None, 14, 14, 96)   0           re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_191 (Lambda)             (None, 14, 14, 96)   0           re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_192 (Lambda)             (None, 14, 14, 96)   0           re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_193 (Lambda)             (None, 14, 14, 96)   0           re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_194 (Lambda)             (None, 14, 14, 96)   0           re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_177 (Conv2D)             (None, 14, 14, 24)   2328        lambda_187[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_178 (Conv2D)             (None, 14, 14, 24)   2328        lambda_188[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_179 (Conv2D)             (None, 14, 14, 24)   2328        lambda_189[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_180 (Conv2D)             (None, 14, 14, 24)   2328        lambda_190[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_181 (Conv2D)             (None, 14, 14, 24)   2328        lambda_191[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_182 (Conv2D)             (None, 14, 14, 24)   2328        lambda_192[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_183 (Conv2D)             (None, 14, 14, 24)   2328        lambda_193[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_184 (Conv2D)             (None, 14, 14, 24)   2328        lambda_194[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_24 (Concatenate)    (None, 14, 14, 192)  0           conv2d_177[0][0]                 \n",
            "                                                                 conv2d_178[0][0]                 \n",
            "                                                                 conv2d_179[0][0]                 \n",
            "                                                                 conv2d_180[0][0]                 \n",
            "                                                                 conv2d_181[0][0]                 \n",
            "                                                                 conv2d_182[0][0]                 \n",
            "                                                                 conv2d_183[0][0]                 \n",
            "                                                                 conv2d_184[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_34 (BatchNo (None, 14, 14, 192)  768         concatenate_24[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_23 (ReLU)                 (None, 14, 14, 192)  0           batch_normalization_34[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_195 (Lambda)             (None, 14, 14, 192)  0           re_lu_23[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_11 (DepthwiseC (None, 14, 14, 192)  1920        lambda_195[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_35 (BatchNo (None, 14, 14, 192)  768         depthwise_conv2d_11[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "lambda_196 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_197 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_198 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_199 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_200 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_201 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_202 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_203 (Lambda)             (None, 14, 14, 24)   0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_185 (Conv2D)             (None, 14, 14, 96)   2400        lambda_196[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_186 (Conv2D)             (None, 14, 14, 96)   2400        lambda_197[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_187 (Conv2D)             (None, 14, 14, 96)   2400        lambda_198[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_188 (Conv2D)             (None, 14, 14, 96)   2400        lambda_199[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_189 (Conv2D)             (None, 14, 14, 96)   2400        lambda_200[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_190 (Conv2D)             (None, 14, 14, 96)   2400        lambda_201[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_191 (Conv2D)             (None, 14, 14, 96)   2400        lambda_202[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_192 (Conv2D)             (None, 14, 14, 96)   2400        lambda_203[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_25 (Concatenate)    (None, 14, 14, 768)  0           conv2d_185[0][0]                 \n",
            "                                                                 conv2d_186[0][0]                 \n",
            "                                                                 conv2d_187[0][0]                 \n",
            "                                                                 conv2d_188[0][0]                 \n",
            "                                                                 conv2d_189[0][0]                 \n",
            "                                                                 conv2d_190[0][0]                 \n",
            "                                                                 conv2d_191[0][0]                 \n",
            "                                                                 conv2d_192[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_36 (BatchNo (None, 14, 14, 768)  3072        concatenate_25[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_9 (Add)                     (None, 14, 14, 768)  0           re_lu_22[0][0]                   \n",
            "                                                                 batch_normalization_36[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_24 (ReLU)                 (None, 14, 14, 768)  0           add_9[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "lambda_204 (Lambda)             (None, 14, 14, 96)   0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_205 (Lambda)             (None, 14, 14, 96)   0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_206 (Lambda)             (None, 14, 14, 96)   0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_207 (Lambda)             (None, 14, 14, 96)   0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_208 (Lambda)             (None, 14, 14, 96)   0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_209 (Lambda)             (None, 14, 14, 96)   0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_210 (Lambda)             (None, 14, 14, 96)   0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_211 (Lambda)             (None, 14, 14, 96)   0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_193 (Conv2D)             (None, 14, 14, 48)   4656        lambda_204[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_194 (Conv2D)             (None, 14, 14, 48)   4656        lambda_205[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_195 (Conv2D)             (None, 14, 14, 48)   4656        lambda_206[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_196 (Conv2D)             (None, 14, 14, 48)   4656        lambda_207[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_197 (Conv2D)             (None, 14, 14, 48)   4656        lambda_208[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_198 (Conv2D)             (None, 14, 14, 48)   4656        lambda_209[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_199 (Conv2D)             (None, 14, 14, 48)   4656        lambda_210[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_200 (Conv2D)             (None, 14, 14, 48)   4656        lambda_211[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_26 (Concatenate)    (None, 14, 14, 384)  0           conv2d_193[0][0]                 \n",
            "                                                                 conv2d_194[0][0]                 \n",
            "                                                                 conv2d_195[0][0]                 \n",
            "                                                                 conv2d_196[0][0]                 \n",
            "                                                                 conv2d_197[0][0]                 \n",
            "                                                                 conv2d_198[0][0]                 \n",
            "                                                                 conv2d_199[0][0]                 \n",
            "                                                                 conv2d_200[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_37 (BatchNo (None, 14, 14, 384)  1536        concatenate_26[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_25 (ReLU)                 (None, 14, 14, 384)  0           batch_normalization_37[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_212 (Lambda)             (None, 14, 14, 384)  0           re_lu_25[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_12 (DepthwiseC (None, 7, 7, 384)    3840        lambda_212[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_38 (BatchNo (None, 7, 7, 384)    1536        depthwise_conv2d_12[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "lambda_213 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_214 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_215 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_216 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_217 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_218 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_219 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_220 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_201 (Conv2D)             (None, 7, 7, 96)     4704        lambda_213[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_202 (Conv2D)             (None, 7, 7, 96)     4704        lambda_214[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_203 (Conv2D)             (None, 7, 7, 96)     4704        lambda_215[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_204 (Conv2D)             (None, 7, 7, 96)     4704        lambda_216[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_205 (Conv2D)             (None, 7, 7, 96)     4704        lambda_217[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_206 (Conv2D)             (None, 7, 7, 96)     4704        lambda_218[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_207 (Conv2D)             (None, 7, 7, 96)     4704        lambda_219[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_208 (Conv2D)             (None, 7, 7, 96)     4704        lambda_220[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_27 (Concatenate)    (None, 7, 7, 768)    0           conv2d_201[0][0]                 \n",
            "                                                                 conv2d_202[0][0]                 \n",
            "                                                                 conv2d_203[0][0]                 \n",
            "                                                                 conv2d_204[0][0]                 \n",
            "                                                                 conv2d_205[0][0]                 \n",
            "                                                                 conv2d_206[0][0]                 \n",
            "                                                                 conv2d_207[0][0]                 \n",
            "                                                                 conv2d_208[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "average_pooling2d_2 (AveragePoo (None, 7, 7, 768)    0           re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_39 (BatchNo (None, 7, 7, 768)    3072        concatenate_27[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_28 (Concatenate)    (None, 7, 7, 1536)   0           average_pooling2d_2[0][0]        \n",
            "                                                                 batch_normalization_39[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_26 (ReLU)                 (None, 7, 7, 1536)   0           concatenate_28[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "lambda_221 (Lambda)             (None, 7, 7, 192)    0           re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_222 (Lambda)             (None, 7, 7, 192)    0           re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_223 (Lambda)             (None, 7, 7, 192)    0           re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_224 (Lambda)             (None, 7, 7, 192)    0           re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_225 (Lambda)             (None, 7, 7, 192)    0           re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_226 (Lambda)             (None, 7, 7, 192)    0           re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_227 (Lambda)             (None, 7, 7, 192)    0           re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_228 (Lambda)             (None, 7, 7, 192)    0           re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_209 (Conv2D)             (None, 7, 7, 48)     9264        lambda_221[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_210 (Conv2D)             (None, 7, 7, 48)     9264        lambda_222[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_211 (Conv2D)             (None, 7, 7, 48)     9264        lambda_223[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_212 (Conv2D)             (None, 7, 7, 48)     9264        lambda_224[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_213 (Conv2D)             (None, 7, 7, 48)     9264        lambda_225[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_214 (Conv2D)             (None, 7, 7, 48)     9264        lambda_226[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_215 (Conv2D)             (None, 7, 7, 48)     9264        lambda_227[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_216 (Conv2D)             (None, 7, 7, 48)     9264        lambda_228[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_29 (Concatenate)    (None, 7, 7, 384)    0           conv2d_209[0][0]                 \n",
            "                                                                 conv2d_210[0][0]                 \n",
            "                                                                 conv2d_211[0][0]                 \n",
            "                                                                 conv2d_212[0][0]                 \n",
            "                                                                 conv2d_213[0][0]                 \n",
            "                                                                 conv2d_214[0][0]                 \n",
            "                                                                 conv2d_215[0][0]                 \n",
            "                                                                 conv2d_216[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_40 (BatchNo (None, 7, 7, 384)    1536        concatenate_29[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_27 (ReLU)                 (None, 7, 7, 384)    0           batch_normalization_40[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_229 (Lambda)             (None, 7, 7, 384)    0           re_lu_27[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_13 (DepthwiseC (None, 7, 7, 384)    3840        lambda_229[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_41 (BatchNo (None, 7, 7, 384)    1536        depthwise_conv2d_13[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "lambda_230 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_231 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_232 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_233 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_234 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_235 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_236 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_237 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_217 (Conv2D)             (None, 7, 7, 192)    9408        lambda_230[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_218 (Conv2D)             (None, 7, 7, 192)    9408        lambda_231[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_219 (Conv2D)             (None, 7, 7, 192)    9408        lambda_232[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_220 (Conv2D)             (None, 7, 7, 192)    9408        lambda_233[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_221 (Conv2D)             (None, 7, 7, 192)    9408        lambda_234[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_222 (Conv2D)             (None, 7, 7, 192)    9408        lambda_235[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_223 (Conv2D)             (None, 7, 7, 192)    9408        lambda_236[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_224 (Conv2D)             (None, 7, 7, 192)    9408        lambda_237[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_30 (Concatenate)    (None, 7, 7, 1536)   0           conv2d_217[0][0]                 \n",
            "                                                                 conv2d_218[0][0]                 \n",
            "                                                                 conv2d_219[0][0]                 \n",
            "                                                                 conv2d_220[0][0]                 \n",
            "                                                                 conv2d_221[0][0]                 \n",
            "                                                                 conv2d_222[0][0]                 \n",
            "                                                                 conv2d_223[0][0]                 \n",
            "                                                                 conv2d_224[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_42 (BatchNo (None, 7, 7, 1536)   6144        concatenate_30[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_10 (Add)                    (None, 7, 7, 1536)   0           re_lu_26[0][0]                   \n",
            "                                                                 batch_normalization_42[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_28 (ReLU)                 (None, 7, 7, 1536)   0           add_10[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_238 (Lambda)             (None, 7, 7, 192)    0           re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_239 (Lambda)             (None, 7, 7, 192)    0           re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_240 (Lambda)             (None, 7, 7, 192)    0           re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_241 (Lambda)             (None, 7, 7, 192)    0           re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_242 (Lambda)             (None, 7, 7, 192)    0           re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_243 (Lambda)             (None, 7, 7, 192)    0           re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_244 (Lambda)             (None, 7, 7, 192)    0           re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_245 (Lambda)             (None, 7, 7, 192)    0           re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_225 (Conv2D)             (None, 7, 7, 48)     9264        lambda_238[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_226 (Conv2D)             (None, 7, 7, 48)     9264        lambda_239[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_227 (Conv2D)             (None, 7, 7, 48)     9264        lambda_240[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_228 (Conv2D)             (None, 7, 7, 48)     9264        lambda_241[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_229 (Conv2D)             (None, 7, 7, 48)     9264        lambda_242[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_230 (Conv2D)             (None, 7, 7, 48)     9264        lambda_243[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_231 (Conv2D)             (None, 7, 7, 48)     9264        lambda_244[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_232 (Conv2D)             (None, 7, 7, 48)     9264        lambda_245[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_31 (Concatenate)    (None, 7, 7, 384)    0           conv2d_225[0][0]                 \n",
            "                                                                 conv2d_226[0][0]                 \n",
            "                                                                 conv2d_227[0][0]                 \n",
            "                                                                 conv2d_228[0][0]                 \n",
            "                                                                 conv2d_229[0][0]                 \n",
            "                                                                 conv2d_230[0][0]                 \n",
            "                                                                 conv2d_231[0][0]                 \n",
            "                                                                 conv2d_232[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_43 (BatchNo (None, 7, 7, 384)    1536        concatenate_31[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_29 (ReLU)                 (None, 7, 7, 384)    0           batch_normalization_43[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_246 (Lambda)             (None, 7, 7, 384)    0           re_lu_29[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_14 (DepthwiseC (None, 7, 7, 384)    3840        lambda_246[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_44 (BatchNo (None, 7, 7, 384)    1536        depthwise_conv2d_14[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "lambda_247 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_248 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_249 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_250 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_251 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_252 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_253 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_254 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_233 (Conv2D)             (None, 7, 7, 192)    9408        lambda_247[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_234 (Conv2D)             (None, 7, 7, 192)    9408        lambda_248[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_235 (Conv2D)             (None, 7, 7, 192)    9408        lambda_249[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_236 (Conv2D)             (None, 7, 7, 192)    9408        lambda_250[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_237 (Conv2D)             (None, 7, 7, 192)    9408        lambda_251[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_238 (Conv2D)             (None, 7, 7, 192)    9408        lambda_252[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_239 (Conv2D)             (None, 7, 7, 192)    9408        lambda_253[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_240 (Conv2D)             (None, 7, 7, 192)    9408        lambda_254[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_32 (Concatenate)    (None, 7, 7, 1536)   0           conv2d_233[0][0]                 \n",
            "                                                                 conv2d_234[0][0]                 \n",
            "                                                                 conv2d_235[0][0]                 \n",
            "                                                                 conv2d_236[0][0]                 \n",
            "                                                                 conv2d_237[0][0]                 \n",
            "                                                                 conv2d_238[0][0]                 \n",
            "                                                                 conv2d_239[0][0]                 \n",
            "                                                                 conv2d_240[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_45 (BatchNo (None, 7, 7, 1536)   6144        concatenate_32[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_11 (Add)                    (None, 7, 7, 1536)   0           re_lu_28[0][0]                   \n",
            "                                                                 batch_normalization_45[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_30 (ReLU)                 (None, 7, 7, 1536)   0           add_11[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_255 (Lambda)             (None, 7, 7, 192)    0           re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_256 (Lambda)             (None, 7, 7, 192)    0           re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_257 (Lambda)             (None, 7, 7, 192)    0           re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_258 (Lambda)             (None, 7, 7, 192)    0           re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_259 (Lambda)             (None, 7, 7, 192)    0           re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_260 (Lambda)             (None, 7, 7, 192)    0           re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_261 (Lambda)             (None, 7, 7, 192)    0           re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lambda_262 (Lambda)             (None, 7, 7, 192)    0           re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_241 (Conv2D)             (None, 7, 7, 48)     9264        lambda_255[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_242 (Conv2D)             (None, 7, 7, 48)     9264        lambda_256[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_243 (Conv2D)             (None, 7, 7, 48)     9264        lambda_257[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_244 (Conv2D)             (None, 7, 7, 48)     9264        lambda_258[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_245 (Conv2D)             (None, 7, 7, 48)     9264        lambda_259[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_246 (Conv2D)             (None, 7, 7, 48)     9264        lambda_260[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_247 (Conv2D)             (None, 7, 7, 48)     9264        lambda_261[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_248 (Conv2D)             (None, 7, 7, 48)     9264        lambda_262[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_33 (Concatenate)    (None, 7, 7, 384)    0           conv2d_241[0][0]                 \n",
            "                                                                 conv2d_242[0][0]                 \n",
            "                                                                 conv2d_243[0][0]                 \n",
            "                                                                 conv2d_244[0][0]                 \n",
            "                                                                 conv2d_245[0][0]                 \n",
            "                                                                 conv2d_246[0][0]                 \n",
            "                                                                 conv2d_247[0][0]                 \n",
            "                                                                 conv2d_248[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_46 (BatchNo (None, 7, 7, 384)    1536        concatenate_33[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_31 (ReLU)                 (None, 7, 7, 384)    0           batch_normalization_46[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_263 (Lambda)             (None, 7, 7, 384)    0           re_lu_31[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "depthwise_conv2d_15 (DepthwiseC (None, 7, 7, 384)    3840        lambda_263[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_47 (BatchNo (None, 7, 7, 384)    1536        depthwise_conv2d_15[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "lambda_264 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_265 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_266 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_267 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_268 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_269 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_270 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "lambda_271 (Lambda)             (None, 7, 7, 48)     0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_249 (Conv2D)             (None, 7, 7, 192)    9408        lambda_264[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_250 (Conv2D)             (None, 7, 7, 192)    9408        lambda_265[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_251 (Conv2D)             (None, 7, 7, 192)    9408        lambda_266[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_252 (Conv2D)             (None, 7, 7, 192)    9408        lambda_267[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_253 (Conv2D)             (None, 7, 7, 192)    9408        lambda_268[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_254 (Conv2D)             (None, 7, 7, 192)    9408        lambda_269[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_255 (Conv2D)             (None, 7, 7, 192)    9408        lambda_270[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_256 (Conv2D)             (None, 7, 7, 192)    9408        lambda_271[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_34 (Concatenate)    (None, 7, 7, 1536)   0           conv2d_249[0][0]                 \n",
            "                                                                 conv2d_250[0][0]                 \n",
            "                                                                 conv2d_251[0][0]                 \n",
            "                                                                 conv2d_252[0][0]                 \n",
            "                                                                 conv2d_253[0][0]                 \n",
            "                                                                 conv2d_254[0][0]                 \n",
            "                                                                 conv2d_255[0][0]                 \n",
            "                                                                 conv2d_256[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_48 (BatchNo (None, 7, 7, 1536)   6144        concatenate_34[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "add_12 (Add)                    (None, 7, 7, 1536)   0           re_lu_30[0][0]                   \n",
            "                                                                 batch_normalization_48[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_32 (ReLU)                 (None, 7, 7, 1536)   0           add_12[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling2d (Globa (None, 1536)         0           re_lu_32[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 1000)         1537000     global_average_pooling2d[0][0]   \n",
            "==================================================================================================\n",
            "Total params: 2,491,504\n",
            "Trainable params: 2,452,336\n",
            "Non-trainable params: 39,168\n",
            "__________________________________________________________________________________________________\n",
            "0.08775172233581544\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "28ghLVhcVfG_",
        "outputId": "96dd4ca0-1a71-43e8-eab6-9b8760836274",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def resnet(input_shape, n_classes):\n",
        "  \n",
        "  def conv_bn_rl(x, f, k=1, s=1, p='same'):\n",
        "    x = Conv2D(f, k, strides=s, padding=p)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def identity_block(tensor, f):\n",
        "    x = conv_bn_rl(tensor, f)\n",
        "    x = conv_bn_rl(x, f, 3)\n",
        "    x = Conv2D(4*f, 1)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    \n",
        "    x = Add()([x, tensor])\n",
        "    output = ReLU()(x)\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def conv_block(tensor, f, s):\n",
        "    x = conv_bn_rl(tensor, f)\n",
        "    x = conv_bn_rl(x, f, 3, s)\n",
        "    x = Conv2D(4*f, 1)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    \n",
        "    shortcut = Conv2D(4*f, 1, strides=s)(tensor)\n",
        "    shortcut = BatchNormalization()(shortcut)\n",
        "    \n",
        "    x = Add()([x, shortcut])\n",
        "    output = ReLU()(x)\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def resnet_block(x, f, r, s=2):\n",
        "    x = conv_block(x, f, s)\n",
        "    for _ in range(r-1):\n",
        "      x = identity_block(x, f)\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = conv_bn_rl(input, 64, 7, 2)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = resnet_block(x, 64, 3, 1)\n",
        "  x = resnet_block(x, 128, 4)\n",
        "  x = resnet_block(x, 256, 6)\n",
        "  x = resnet_block(x, 512, 3)\n",
        "  \n",
        "  x = GlobalAvgPool2D()(x)\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model\n",
        "\n",
        "\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = resnet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d (Conv2D)                 (None, 112, 112, 64) 9472        input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization (BatchNorma (None, 112, 112, 64) 256         conv2d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu (ReLU)                    (None, 112, 112, 64) 0           batch_normalization[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D)    (None, 56, 56, 64)   0           re_lu[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 56, 56, 64)   4160        max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 56, 56, 64)   256         conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_1 (ReLU)                  (None, 56, 56, 64)   0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 56, 56, 64)   36928       re_lu_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 56, 56, 64)   256         conv2d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_2 (ReLU)                  (None, 56, 56, 64)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 56, 56, 256)  16640       re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 56, 56, 256)  16640       max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 56, 56, 256)  1024        conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 56, 56, 256)  1024        conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "add (Add)                       (None, 56, 56, 256)  0           batch_normalization_3[0][0]      \n",
            "                                                                 batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_3 (ReLU)                  (None, 56, 56, 256)  0           add[0][0]                        \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 56, 56, 64)   16448       re_lu_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 56, 56, 64)   256         conv2d_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_4 (ReLU)                  (None, 56, 56, 64)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_6 (Conv2D)               (None, 56, 56, 64)   36928       re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 56, 56, 64)   256         conv2d_6[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_5 (ReLU)                  (None, 56, 56, 64)   0           batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_7 (Conv2D)               (None, 56, 56, 256)  16640       re_lu_5[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_7 (BatchNor (None, 56, 56, 256)  1024        conv2d_7[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "add_1 (Add)                     (None, 56, 56, 256)  0           batch_normalization_7[0][0]      \n",
            "                                                                 re_lu_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_6 (ReLU)                  (None, 56, 56, 256)  0           add_1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_8 (Conv2D)               (None, 56, 56, 64)   16448       re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_8 (BatchNor (None, 56, 56, 64)   256         conv2d_8[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_7 (ReLU)                  (None, 56, 56, 64)   0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 56, 56, 64)   36928       re_lu_7[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_9 (BatchNor (None, 56, 56, 64)   256         conv2d_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_8 (ReLU)                  (None, 56, 56, 64)   0           batch_normalization_9[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 56, 56, 256)  16640       re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_10 (BatchNo (None, 56, 56, 256)  1024        conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_2 (Add)                     (None, 56, 56, 256)  0           batch_normalization_10[0][0]     \n",
            "                                                                 re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_9 (ReLU)                  (None, 56, 56, 256)  0           add_2[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 56, 56, 128)  32896       re_lu_9[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_11 (BatchNo (None, 56, 56, 128)  512         conv2d_11[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_10 (ReLU)                 (None, 56, 56, 128)  0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_12 (Conv2D)              (None, 28, 28, 128)  147584      re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_12 (BatchNo (None, 28, 28, 128)  512         conv2d_12[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_11 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_12[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_13 (Conv2D)              (None, 28, 28, 512)  66048       re_lu_11[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_14 (Conv2D)              (None, 28, 28, 512)  131584      re_lu_9[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_13 (BatchNo (None, 28, 28, 512)  2048        conv2d_13[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_14 (BatchNo (None, 28, 28, 512)  2048        conv2d_14[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_3 (Add)                     (None, 28, 28, 512)  0           batch_normalization_13[0][0]     \n",
            "                                                                 batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_12 (ReLU)                 (None, 28, 28, 512)  0           add_3[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_15 (Conv2D)              (None, 28, 28, 128)  65664       re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_15 (BatchNo (None, 28, 28, 128)  512         conv2d_15[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_13 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_15[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_16 (Conv2D)              (None, 28, 28, 128)  147584      re_lu_13[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_16 (BatchNo (None, 28, 28, 128)  512         conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_14 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_16[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_17 (Conv2D)              (None, 28, 28, 512)  66048       re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_17 (BatchNo (None, 28, 28, 512)  2048        conv2d_17[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_4 (Add)                     (None, 28, 28, 512)  0           batch_normalization_17[0][0]     \n",
            "                                                                 re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_15 (ReLU)                 (None, 28, 28, 512)  0           add_4[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_18 (Conv2D)              (None, 28, 28, 128)  65664       re_lu_15[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_18 (BatchNo (None, 28, 28, 128)  512         conv2d_18[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_16 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_18[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_19 (Conv2D)              (None, 28, 28, 128)  147584      re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_19 (BatchNo (None, 28, 28, 128)  512         conv2d_19[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_17 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_19[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_20 (Conv2D)              (None, 28, 28, 512)  66048       re_lu_17[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_20 (BatchNo (None, 28, 28, 512)  2048        conv2d_20[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_5 (Add)                     (None, 28, 28, 512)  0           batch_normalization_20[0][0]     \n",
            "                                                                 re_lu_15[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_18 (ReLU)                 (None, 28, 28, 512)  0           add_5[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_21 (Conv2D)              (None, 28, 28, 128)  65664       re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_21 (BatchNo (None, 28, 28, 128)  512         conv2d_21[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_19 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_21[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_22 (Conv2D)              (None, 28, 28, 128)  147584      re_lu_19[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_22 (BatchNo (None, 28, 28, 128)  512         conv2d_22[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_20 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_22[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_23 (Conv2D)              (None, 28, 28, 512)  66048       re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_23 (BatchNo (None, 28, 28, 512)  2048        conv2d_23[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_6 (Add)                     (None, 28, 28, 512)  0           batch_normalization_23[0][0]     \n",
            "                                                                 re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_21 (ReLU)                 (None, 28, 28, 512)  0           add_6[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_24 (Conv2D)              (None, 28, 28, 256)  131328      re_lu_21[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_24 (BatchNo (None, 28, 28, 256)  1024        conv2d_24[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_22 (ReLU)                 (None, 28, 28, 256)  0           batch_normalization_24[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_25 (Conv2D)              (None, 14, 14, 256)  590080      re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_25 (BatchNo (None, 14, 14, 256)  1024        conv2d_25[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_23 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_25[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_26 (Conv2D)              (None, 14, 14, 1024) 263168      re_lu_23[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_27 (Conv2D)              (None, 14, 14, 1024) 525312      re_lu_21[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_26 (BatchNo (None, 14, 14, 1024) 4096        conv2d_26[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_27 (BatchNo (None, 14, 14, 1024) 4096        conv2d_27[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_7 (Add)                     (None, 14, 14, 1024) 0           batch_normalization_26[0][0]     \n",
            "                                                                 batch_normalization_27[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_24 (ReLU)                 (None, 14, 14, 1024) 0           add_7[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_28 (Conv2D)              (None, 14, 14, 256)  262400      re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_28 (BatchNo (None, 14, 14, 256)  1024        conv2d_28[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_25 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_28[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_29 (Conv2D)              (None, 14, 14, 256)  590080      re_lu_25[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_29 (BatchNo (None, 14, 14, 256)  1024        conv2d_29[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_26 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_30 (Conv2D)              (None, 14, 14, 1024) 263168      re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_30 (BatchNo (None, 14, 14, 1024) 4096        conv2d_30[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_8 (Add)                     (None, 14, 14, 1024) 0           batch_normalization_30[0][0]     \n",
            "                                                                 re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_27 (ReLU)                 (None, 14, 14, 1024) 0           add_8[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_31 (Conv2D)              (None, 14, 14, 256)  262400      re_lu_27[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_31 (BatchNo (None, 14, 14, 256)  1024        conv2d_31[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_28 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_31[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_32 (Conv2D)              (None, 14, 14, 256)  590080      re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_32 (BatchNo (None, 14, 14, 256)  1024        conv2d_32[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_29 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_33 (Conv2D)              (None, 14, 14, 1024) 263168      re_lu_29[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_33 (BatchNo (None, 14, 14, 1024) 4096        conv2d_33[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_9 (Add)                     (None, 14, 14, 1024) 0           batch_normalization_33[0][0]     \n",
            "                                                                 re_lu_27[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_30 (ReLU)                 (None, 14, 14, 1024) 0           add_9[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_34 (Conv2D)              (None, 14, 14, 256)  262400      re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_34 (BatchNo (None, 14, 14, 256)  1024        conv2d_34[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_31 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_34[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_35 (Conv2D)              (None, 14, 14, 256)  590080      re_lu_31[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_35 (BatchNo (None, 14, 14, 256)  1024        conv2d_35[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_32 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_36 (Conv2D)              (None, 14, 14, 1024) 263168      re_lu_32[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_36 (BatchNo (None, 14, 14, 1024) 4096        conv2d_36[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_10 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_36[0][0]     \n",
            "                                                                 re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_33 (ReLU)                 (None, 14, 14, 1024) 0           add_10[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_37 (Conv2D)              (None, 14, 14, 256)  262400      re_lu_33[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_37 (BatchNo (None, 14, 14, 256)  1024        conv2d_37[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_34 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_37[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_38 (Conv2D)              (None, 14, 14, 256)  590080      re_lu_34[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_38 (BatchNo (None, 14, 14, 256)  1024        conv2d_38[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_35 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_39 (Conv2D)              (None, 14, 14, 1024) 263168      re_lu_35[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_39 (BatchNo (None, 14, 14, 1024) 4096        conv2d_39[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_11 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_39[0][0]     \n",
            "                                                                 re_lu_33[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_36 (ReLU)                 (None, 14, 14, 1024) 0           add_11[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_40 (Conv2D)              (None, 14, 14, 256)  262400      re_lu_36[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_40 (BatchNo (None, 14, 14, 256)  1024        conv2d_40[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_37 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_40[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_41 (Conv2D)              (None, 14, 14, 256)  590080      re_lu_37[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_41 (BatchNo (None, 14, 14, 256)  1024        conv2d_41[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_38 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_42 (Conv2D)              (None, 14, 14, 1024) 263168      re_lu_38[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_42 (BatchNo (None, 14, 14, 1024) 4096        conv2d_42[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_12 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_42[0][0]     \n",
            "                                                                 re_lu_36[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_39 (ReLU)                 (None, 14, 14, 1024) 0           add_12[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_43 (Conv2D)              (None, 14, 14, 512)  524800      re_lu_39[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_43 (BatchNo (None, 14, 14, 512)  2048        conv2d_43[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_40 (ReLU)                 (None, 14, 14, 512)  0           batch_normalization_43[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_44 (Conv2D)              (None, 7, 7, 512)    2359808     re_lu_40[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_44 (BatchNo (None, 7, 7, 512)    2048        conv2d_44[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_41 (ReLU)                 (None, 7, 7, 512)    0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_45 (Conv2D)              (None, 7, 7, 2048)   1050624     re_lu_41[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_46 (Conv2D)              (None, 7, 7, 2048)   2099200     re_lu_39[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_45 (BatchNo (None, 7, 7, 2048)   8192        conv2d_45[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_46 (BatchNo (None, 7, 7, 2048)   8192        conv2d_46[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_13 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_45[0][0]     \n",
            "                                                                 batch_normalization_46[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_42 (ReLU)                 (None, 7, 7, 2048)   0           add_13[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_47 (Conv2D)              (None, 7, 7, 512)    1049088     re_lu_42[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_47 (BatchNo (None, 7, 7, 512)    2048        conv2d_47[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_43 (ReLU)                 (None, 7, 7, 512)    0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_48 (Conv2D)              (None, 7, 7, 512)    2359808     re_lu_43[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_48 (BatchNo (None, 7, 7, 512)    2048        conv2d_48[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_44 (ReLU)                 (None, 7, 7, 512)    0           batch_normalization_48[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_49 (Conv2D)              (None, 7, 7, 2048)   1050624     re_lu_44[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_49 (BatchNo (None, 7, 7, 2048)   8192        conv2d_49[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_14 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_49[0][0]     \n",
            "                                                                 re_lu_42[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_45 (ReLU)                 (None, 7, 7, 2048)   0           add_14[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_50 (Conv2D)              (None, 7, 7, 512)    1049088     re_lu_45[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_50 (BatchNo (None, 7, 7, 512)    2048        conv2d_50[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_46 (ReLU)                 (None, 7, 7, 512)    0           batch_normalization_50[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_51 (Conv2D)              (None, 7, 7, 512)    2359808     re_lu_46[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_51 (BatchNo (None, 7, 7, 512)    2048        conv2d_51[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_47 (ReLU)                 (None, 7, 7, 512)    0           batch_normalization_51[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_52 (Conv2D)              (None, 7, 7, 2048)   1050624     re_lu_47[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_52 (BatchNo (None, 7, 7, 2048)   8192        conv2d_52[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_15 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_52[0][0]     \n",
            "                                                                 re_lu_45[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_48 (ReLU)                 (None, 7, 7, 2048)   0           add_15[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling2d (Globa (None, 2048)         0           re_lu_48[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 1000)         2049000     global_average_pooling2d[0][0]   \n",
            "==================================================================================================\n",
            "Total params: 25,636,712\n",
            "Trainable params: 25,583,592\n",
            "Non-trainable params: 53,120\n",
            "__________________________________________________________________________________________________\n",
            "0.17680606842041016\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHiBEBOQVv_R",
        "outputId": "e2b35272-4eab-4a04-c3f8-0a513fb6f62e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def densenet(img_shape, n_classes, f=32):\n",
        "  repetitions = 6, 12, 24, 16\n",
        "  \n",
        "  def bn_rl_conv(x, f, k=1, s=1, p='same'):\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    x = Conv2D(f, k, strides=s, padding=p)(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def dense_block(tensor, r):\n",
        "    for _ in range(r):\n",
        "      x = bn_rl_conv(tensor, 4*f)\n",
        "      x = bn_rl_conv(x, f, 3)\n",
        "      tensor = Concatenate()([tensor, x])\n",
        "    return tensor\n",
        "  \n",
        "  \n",
        "  def transition_block(x):\n",
        "    x = bn_rl_conv(x, K.int_shape(x)[-1] // 2)\n",
        "    x = AvgPool2D(2, strides=2, padding='same')(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  input = Input(img_shape)\n",
        "  \n",
        "  x = Conv2D(64, 7, strides=2, padding='same')(input)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  for r in repetitions:\n",
        "    d = dense_block(x, r)\n",
        "    x = transition_block(d)\n",
        "  \n",
        "  x = GlobalAvgPool2D()(d)\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model\n",
        "\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = densenet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d (Conv2D)                 (None, 112, 112, 64) 9472        input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D)    (None, 56, 56, 64)   0           conv2d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization (BatchNorma (None, 56, 56, 64)   256         max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu (ReLU)                    (None, 56, 56, 64)   0           batch_normalization[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 56, 56, 128)  8320        re_lu[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 56, 56, 128)  512         conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_1 (ReLU)                  (None, 56, 56, 128)  0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 56, 56, 32)   36896       re_lu_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 56, 56, 96)   0           max_pooling2d[0][0]              \n",
            "                                                                 conv2d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 56, 56, 96)   384         concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_2 (ReLU)                  (None, 56, 56, 96)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 56, 56, 128)  12416       re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 56, 56, 128)  512         conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_3 (ReLU)                  (None, 56, 56, 128)  0           batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 56, 56, 32)   36896       re_lu_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 56, 56, 128)  0           concatenate[0][0]                \n",
            "                                                                 conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 56, 56, 128)  512         concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_4 (ReLU)                  (None, 56, 56, 128)  0           batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 56, 56, 128)  16512       re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 56, 56, 128)  512         conv2d_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_5 (ReLU)                  (None, 56, 56, 128)  0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_6 (Conv2D)               (None, 56, 56, 32)   36896       re_lu_5[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_2 (Concatenate)     (None, 56, 56, 160)  0           concatenate_1[0][0]              \n",
            "                                                                 conv2d_6[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 56, 56, 160)  640         concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_6 (ReLU)                  (None, 56, 56, 160)  0           batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_7 (Conv2D)               (None, 56, 56, 128)  20608       re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_7 (BatchNor (None, 56, 56, 128)  512         conv2d_7[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_7 (ReLU)                  (None, 56, 56, 128)  0           batch_normalization_7[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_8 (Conv2D)               (None, 56, 56, 32)   36896       re_lu_7[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)     (None, 56, 56, 192)  0           concatenate_2[0][0]              \n",
            "                                                                 conv2d_8[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_8 (BatchNor (None, 56, 56, 192)  768         concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_8 (ReLU)                  (None, 56, 56, 192)  0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 56, 56, 128)  24704       re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_9 (BatchNor (None, 56, 56, 128)  512         conv2d_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_9 (ReLU)                  (None, 56, 56, 128)  0           batch_normalization_9[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 56, 56, 32)   36896       re_lu_9[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_4 (Concatenate)     (None, 56, 56, 224)  0           concatenate_3[0][0]              \n",
            "                                                                 conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_10 (BatchNo (None, 56, 56, 224)  896         concatenate_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_10 (ReLU)                 (None, 56, 56, 224)  0           batch_normalization_10[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 56, 56, 128)  28800       re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_11 (BatchNo (None, 56, 56, 128)  512         conv2d_11[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_11 (ReLU)                 (None, 56, 56, 128)  0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_12 (Conv2D)              (None, 56, 56, 32)   36896       re_lu_11[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_5 (Concatenate)     (None, 56, 56, 256)  0           concatenate_4[0][0]              \n",
            "                                                                 conv2d_12[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_12 (BatchNo (None, 56, 56, 256)  1024        concatenate_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_12 (ReLU)                 (None, 56, 56, 256)  0           batch_normalization_12[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_13 (Conv2D)              (None, 56, 56, 128)  32896       re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "average_pooling2d (AveragePooli (None, 28, 28, 128)  0           conv2d_13[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_13 (BatchNo (None, 28, 28, 128)  512         average_pooling2d[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_13 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_13[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_14 (Conv2D)              (None, 28, 28, 128)  16512       re_lu_13[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_14 (BatchNo (None, 28, 28, 128)  512         conv2d_14[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_14 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_15 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_6 (Concatenate)     (None, 28, 28, 160)  0           average_pooling2d[0][0]          \n",
            "                                                                 conv2d_15[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_15 (BatchNo (None, 28, 28, 160)  640         concatenate_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_15 (ReLU)                 (None, 28, 28, 160)  0           batch_normalization_15[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_16 (Conv2D)              (None, 28, 28, 128)  20608       re_lu_15[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_16 (BatchNo (None, 28, 28, 128)  512         conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_16 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_16[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_17 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_7 (Concatenate)     (None, 28, 28, 192)  0           concatenate_6[0][0]              \n",
            "                                                                 conv2d_17[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_17 (BatchNo (None, 28, 28, 192)  768         concatenate_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_17 (ReLU)                 (None, 28, 28, 192)  0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_18 (Conv2D)              (None, 28, 28, 128)  24704       re_lu_17[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_18 (BatchNo (None, 28, 28, 128)  512         conv2d_18[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_18 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_18[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_19 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_8 (Concatenate)     (None, 28, 28, 224)  0           concatenate_7[0][0]              \n",
            "                                                                 conv2d_19[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_19 (BatchNo (None, 28, 28, 224)  896         concatenate_8[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_19 (ReLU)                 (None, 28, 28, 224)  0           batch_normalization_19[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_20 (Conv2D)              (None, 28, 28, 128)  28800       re_lu_19[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_20 (BatchNo (None, 28, 28, 128)  512         conv2d_20[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_20 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_21 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_9 (Concatenate)     (None, 28, 28, 256)  0           concatenate_8[0][0]              \n",
            "                                                                 conv2d_21[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_21 (BatchNo (None, 28, 28, 256)  1024        concatenate_9[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_21 (ReLU)                 (None, 28, 28, 256)  0           batch_normalization_21[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_22 (Conv2D)              (None, 28, 28, 128)  32896       re_lu_21[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_22 (BatchNo (None, 28, 28, 128)  512         conv2d_22[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_22 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_22[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_23 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_10 (Concatenate)    (None, 28, 28, 288)  0           concatenate_9[0][0]              \n",
            "                                                                 conv2d_23[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_23 (BatchNo (None, 28, 28, 288)  1152        concatenate_10[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_23 (ReLU)                 (None, 28, 28, 288)  0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_24 (Conv2D)              (None, 28, 28, 128)  36992       re_lu_23[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_24 (BatchNo (None, 28, 28, 128)  512         conv2d_24[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_24 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_24[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_25 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_11 (Concatenate)    (None, 28, 28, 320)  0           concatenate_10[0][0]             \n",
            "                                                                 conv2d_25[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_25 (BatchNo (None, 28, 28, 320)  1280        concatenate_11[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_25 (ReLU)                 (None, 28, 28, 320)  0           batch_normalization_25[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_26 (Conv2D)              (None, 28, 28, 128)  41088       re_lu_25[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_26 (BatchNo (None, 28, 28, 128)  512         conv2d_26[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_26 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_27 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_12 (Concatenate)    (None, 28, 28, 352)  0           concatenate_11[0][0]             \n",
            "                                                                 conv2d_27[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_27 (BatchNo (None, 28, 28, 352)  1408        concatenate_12[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_27 (ReLU)                 (None, 28, 28, 352)  0           batch_normalization_27[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_28 (Conv2D)              (None, 28, 28, 128)  45184       re_lu_27[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_28 (BatchNo (None, 28, 28, 128)  512         conv2d_28[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_28 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_28[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_29 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_13 (Concatenate)    (None, 28, 28, 384)  0           concatenate_12[0][0]             \n",
            "                                                                 conv2d_29[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_29 (BatchNo (None, 28, 28, 384)  1536        concatenate_13[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_29 (ReLU)                 (None, 28, 28, 384)  0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_30 (Conv2D)              (None, 28, 28, 128)  49280       re_lu_29[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_30 (BatchNo (None, 28, 28, 128)  512         conv2d_30[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_30 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_30[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_31 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_14 (Concatenate)    (None, 28, 28, 416)  0           concatenate_13[0][0]             \n",
            "                                                                 conv2d_31[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_31 (BatchNo (None, 28, 28, 416)  1664        concatenate_14[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_31 (ReLU)                 (None, 28, 28, 416)  0           batch_normalization_31[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_32 (Conv2D)              (None, 28, 28, 128)  53376       re_lu_31[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_32 (BatchNo (None, 28, 28, 128)  512         conv2d_32[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_32 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_33 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_32[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_15 (Concatenate)    (None, 28, 28, 448)  0           concatenate_14[0][0]             \n",
            "                                                                 conv2d_33[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_33 (BatchNo (None, 28, 28, 448)  1792        concatenate_15[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_33 (ReLU)                 (None, 28, 28, 448)  0           batch_normalization_33[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_34 (Conv2D)              (None, 28, 28, 128)  57472       re_lu_33[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_34 (BatchNo (None, 28, 28, 128)  512         conv2d_34[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_34 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_34[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_35 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_34[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_16 (Concatenate)    (None, 28, 28, 480)  0           concatenate_15[0][0]             \n",
            "                                                                 conv2d_35[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_35 (BatchNo (None, 28, 28, 480)  1920        concatenate_16[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_35 (ReLU)                 (None, 28, 28, 480)  0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_36 (Conv2D)              (None, 28, 28, 128)  61568       re_lu_35[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_36 (BatchNo (None, 28, 28, 128)  512         conv2d_36[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_36 (ReLU)                 (None, 28, 28, 128)  0           batch_normalization_36[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_37 (Conv2D)              (None, 28, 28, 32)   36896       re_lu_36[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_17 (Concatenate)    (None, 28, 28, 512)  0           concatenate_16[0][0]             \n",
            "                                                                 conv2d_37[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_37 (BatchNo (None, 28, 28, 512)  2048        concatenate_17[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_37 (ReLU)                 (None, 28, 28, 512)  0           batch_normalization_37[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_38 (Conv2D)              (None, 28, 28, 256)  131328      re_lu_37[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "average_pooling2d_1 (AveragePoo (None, 14, 14, 256)  0           conv2d_38[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_38 (BatchNo (None, 14, 14, 256)  1024        average_pooling2d_1[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_38 (ReLU)                 (None, 14, 14, 256)  0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_39 (Conv2D)              (None, 14, 14, 128)  32896       re_lu_38[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_39 (BatchNo (None, 14, 14, 128)  512         conv2d_39[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_39 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_39[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_40 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_39[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_18 (Concatenate)    (None, 14, 14, 288)  0           average_pooling2d_1[0][0]        \n",
            "                                                                 conv2d_40[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_40 (BatchNo (None, 14, 14, 288)  1152        concatenate_18[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_40 (ReLU)                 (None, 14, 14, 288)  0           batch_normalization_40[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_41 (Conv2D)              (None, 14, 14, 128)  36992       re_lu_40[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_41 (BatchNo (None, 14, 14, 128)  512         conv2d_41[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_41 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_41[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_42 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_41[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_19 (Concatenate)    (None, 14, 14, 320)  0           concatenate_18[0][0]             \n",
            "                                                                 conv2d_42[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_42 (BatchNo (None, 14, 14, 320)  1280        concatenate_19[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_42 (ReLU)                 (None, 14, 14, 320)  0           batch_normalization_42[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_43 (Conv2D)              (None, 14, 14, 128)  41088       re_lu_42[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_43 (BatchNo (None, 14, 14, 128)  512         conv2d_43[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_43 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_43[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_44 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_43[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_20 (Concatenate)    (None, 14, 14, 352)  0           concatenate_19[0][0]             \n",
            "                                                                 conv2d_44[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_44 (BatchNo (None, 14, 14, 352)  1408        concatenate_20[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_44 (ReLU)                 (None, 14, 14, 352)  0           batch_normalization_44[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_45 (Conv2D)              (None, 14, 14, 128)  45184       re_lu_44[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_45 (BatchNo (None, 14, 14, 128)  512         conv2d_45[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_45 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_45[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_46 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_45[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_21 (Concatenate)    (None, 14, 14, 384)  0           concatenate_20[0][0]             \n",
            "                                                                 conv2d_46[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_46 (BatchNo (None, 14, 14, 384)  1536        concatenate_21[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_46 (ReLU)                 (None, 14, 14, 384)  0           batch_normalization_46[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_47 (Conv2D)              (None, 14, 14, 128)  49280       re_lu_46[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_47 (BatchNo (None, 14, 14, 128)  512         conv2d_47[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_47 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_47[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_48 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_47[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_22 (Concatenate)    (None, 14, 14, 416)  0           concatenate_21[0][0]             \n",
            "                                                                 conv2d_48[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_48 (BatchNo (None, 14, 14, 416)  1664        concatenate_22[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_48 (ReLU)                 (None, 14, 14, 416)  0           batch_normalization_48[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_49 (Conv2D)              (None, 14, 14, 128)  53376       re_lu_48[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_49 (BatchNo (None, 14, 14, 128)  512         conv2d_49[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_49 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_49[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_50 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_49[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_23 (Concatenate)    (None, 14, 14, 448)  0           concatenate_22[0][0]             \n",
            "                                                                 conv2d_50[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_50 (BatchNo (None, 14, 14, 448)  1792        concatenate_23[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_50 (ReLU)                 (None, 14, 14, 448)  0           batch_normalization_50[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_51 (Conv2D)              (None, 14, 14, 128)  57472       re_lu_50[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_51 (BatchNo (None, 14, 14, 128)  512         conv2d_51[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_51 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_51[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_52 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_51[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_24 (Concatenate)    (None, 14, 14, 480)  0           concatenate_23[0][0]             \n",
            "                                                                 conv2d_52[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_52 (BatchNo (None, 14, 14, 480)  1920        concatenate_24[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_52 (ReLU)                 (None, 14, 14, 480)  0           batch_normalization_52[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_53 (Conv2D)              (None, 14, 14, 128)  61568       re_lu_52[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_53 (BatchNo (None, 14, 14, 128)  512         conv2d_53[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_53 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_53[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_54 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_53[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_25 (Concatenate)    (None, 14, 14, 512)  0           concatenate_24[0][0]             \n",
            "                                                                 conv2d_54[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_54 (BatchNo (None, 14, 14, 512)  2048        concatenate_25[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_54 (ReLU)                 (None, 14, 14, 512)  0           batch_normalization_54[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_55 (Conv2D)              (None, 14, 14, 128)  65664       re_lu_54[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_55 (BatchNo (None, 14, 14, 128)  512         conv2d_55[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_55 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_55[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_56 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_55[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_26 (Concatenate)    (None, 14, 14, 544)  0           concatenate_25[0][0]             \n",
            "                                                                 conv2d_56[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_56 (BatchNo (None, 14, 14, 544)  2176        concatenate_26[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_56 (ReLU)                 (None, 14, 14, 544)  0           batch_normalization_56[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_57 (Conv2D)              (None, 14, 14, 128)  69760       re_lu_56[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_57 (BatchNo (None, 14, 14, 128)  512         conv2d_57[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_57 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_57[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_58 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_57[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_27 (Concatenate)    (None, 14, 14, 576)  0           concatenate_26[0][0]             \n",
            "                                                                 conv2d_58[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_58 (BatchNo (None, 14, 14, 576)  2304        concatenate_27[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_58 (ReLU)                 (None, 14, 14, 576)  0           batch_normalization_58[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_59 (Conv2D)              (None, 14, 14, 128)  73856       re_lu_58[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_59 (BatchNo (None, 14, 14, 128)  512         conv2d_59[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_59 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_59[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_60 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_59[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_28 (Concatenate)    (None, 14, 14, 608)  0           concatenate_27[0][0]             \n",
            "                                                                 conv2d_60[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_60 (BatchNo (None, 14, 14, 608)  2432        concatenate_28[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_60 (ReLU)                 (None, 14, 14, 608)  0           batch_normalization_60[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_61 (Conv2D)              (None, 14, 14, 128)  77952       re_lu_60[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_61 (BatchNo (None, 14, 14, 128)  512         conv2d_61[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_61 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_61[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_62 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_61[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_29 (Concatenate)    (None, 14, 14, 640)  0           concatenate_28[0][0]             \n",
            "                                                                 conv2d_62[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_62 (BatchNo (None, 14, 14, 640)  2560        concatenate_29[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_62 (ReLU)                 (None, 14, 14, 640)  0           batch_normalization_62[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_63 (Conv2D)              (None, 14, 14, 128)  82048       re_lu_62[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_63 (BatchNo (None, 14, 14, 128)  512         conv2d_63[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_63 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_63[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_64 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_63[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_30 (Concatenate)    (None, 14, 14, 672)  0           concatenate_29[0][0]             \n",
            "                                                                 conv2d_64[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_64 (BatchNo (None, 14, 14, 672)  2688        concatenate_30[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_64 (ReLU)                 (None, 14, 14, 672)  0           batch_normalization_64[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_65 (Conv2D)              (None, 14, 14, 128)  86144       re_lu_64[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_65 (BatchNo (None, 14, 14, 128)  512         conv2d_65[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_65 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_65[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_66 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_65[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_31 (Concatenate)    (None, 14, 14, 704)  0           concatenate_30[0][0]             \n",
            "                                                                 conv2d_66[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_66 (BatchNo (None, 14, 14, 704)  2816        concatenate_31[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_66 (ReLU)                 (None, 14, 14, 704)  0           batch_normalization_66[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_67 (Conv2D)              (None, 14, 14, 128)  90240       re_lu_66[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_67 (BatchNo (None, 14, 14, 128)  512         conv2d_67[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_67 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_67[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_68 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_67[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_32 (Concatenate)    (None, 14, 14, 736)  0           concatenate_31[0][0]             \n",
            "                                                                 conv2d_68[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_68 (BatchNo (None, 14, 14, 736)  2944        concatenate_32[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_68 (ReLU)                 (None, 14, 14, 736)  0           batch_normalization_68[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_69 (Conv2D)              (None, 14, 14, 128)  94336       re_lu_68[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_69 (BatchNo (None, 14, 14, 128)  512         conv2d_69[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_69 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_69[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_70 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_69[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_33 (Concatenate)    (None, 14, 14, 768)  0           concatenate_32[0][0]             \n",
            "                                                                 conv2d_70[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_70 (BatchNo (None, 14, 14, 768)  3072        concatenate_33[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_70 (ReLU)                 (None, 14, 14, 768)  0           batch_normalization_70[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_71 (Conv2D)              (None, 14, 14, 128)  98432       re_lu_70[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_71 (BatchNo (None, 14, 14, 128)  512         conv2d_71[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_71 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_71[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_72 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_71[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_34 (Concatenate)    (None, 14, 14, 800)  0           concatenate_33[0][0]             \n",
            "                                                                 conv2d_72[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_72 (BatchNo (None, 14, 14, 800)  3200        concatenate_34[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_72 (ReLU)                 (None, 14, 14, 800)  0           batch_normalization_72[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_73 (Conv2D)              (None, 14, 14, 128)  102528      re_lu_72[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_73 (BatchNo (None, 14, 14, 128)  512         conv2d_73[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_73 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_73[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_74 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_73[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_35 (Concatenate)    (None, 14, 14, 832)  0           concatenate_34[0][0]             \n",
            "                                                                 conv2d_74[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_74 (BatchNo (None, 14, 14, 832)  3328        concatenate_35[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_74 (ReLU)                 (None, 14, 14, 832)  0           batch_normalization_74[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_75 (Conv2D)              (None, 14, 14, 128)  106624      re_lu_74[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_75 (BatchNo (None, 14, 14, 128)  512         conv2d_75[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_75 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_75[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_76 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_75[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_36 (Concatenate)    (None, 14, 14, 864)  0           concatenate_35[0][0]             \n",
            "                                                                 conv2d_76[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_76 (BatchNo (None, 14, 14, 864)  3456        concatenate_36[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_76 (ReLU)                 (None, 14, 14, 864)  0           batch_normalization_76[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_77 (Conv2D)              (None, 14, 14, 128)  110720      re_lu_76[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_77 (BatchNo (None, 14, 14, 128)  512         conv2d_77[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_77 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_77[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_78 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_77[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_37 (Concatenate)    (None, 14, 14, 896)  0           concatenate_36[0][0]             \n",
            "                                                                 conv2d_78[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_78 (BatchNo (None, 14, 14, 896)  3584        concatenate_37[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_78 (ReLU)                 (None, 14, 14, 896)  0           batch_normalization_78[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_79 (Conv2D)              (None, 14, 14, 128)  114816      re_lu_78[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_79 (BatchNo (None, 14, 14, 128)  512         conv2d_79[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_79 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_79[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_80 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_79[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_38 (Concatenate)    (None, 14, 14, 928)  0           concatenate_37[0][0]             \n",
            "                                                                 conv2d_80[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_80 (BatchNo (None, 14, 14, 928)  3712        concatenate_38[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_80 (ReLU)                 (None, 14, 14, 928)  0           batch_normalization_80[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_81 (Conv2D)              (None, 14, 14, 128)  118912      re_lu_80[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_81 (BatchNo (None, 14, 14, 128)  512         conv2d_81[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_81 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_81[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_82 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_81[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_39 (Concatenate)    (None, 14, 14, 960)  0           concatenate_38[0][0]             \n",
            "                                                                 conv2d_82[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_82 (BatchNo (None, 14, 14, 960)  3840        concatenate_39[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_82 (ReLU)                 (None, 14, 14, 960)  0           batch_normalization_82[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_83 (Conv2D)              (None, 14, 14, 128)  123008      re_lu_82[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_83 (BatchNo (None, 14, 14, 128)  512         conv2d_83[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_83 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_83[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_84 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_83[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_40 (Concatenate)    (None, 14, 14, 992)  0           concatenate_39[0][0]             \n",
            "                                                                 conv2d_84[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_84 (BatchNo (None, 14, 14, 992)  3968        concatenate_40[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_84 (ReLU)                 (None, 14, 14, 992)  0           batch_normalization_84[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_85 (Conv2D)              (None, 14, 14, 128)  127104      re_lu_84[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_85 (BatchNo (None, 14, 14, 128)  512         conv2d_85[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_85 (ReLU)                 (None, 14, 14, 128)  0           batch_normalization_85[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_86 (Conv2D)              (None, 14, 14, 32)   36896       re_lu_85[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_41 (Concatenate)    (None, 14, 14, 1024) 0           concatenate_40[0][0]             \n",
            "                                                                 conv2d_86[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_86 (BatchNo (None, 14, 14, 1024) 4096        concatenate_41[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_86 (ReLU)                 (None, 14, 14, 1024) 0           batch_normalization_86[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_87 (Conv2D)              (None, 14, 14, 512)  524800      re_lu_86[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "average_pooling2d_2 (AveragePoo (None, 7, 7, 512)    0           conv2d_87[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_87 (BatchNo (None, 7, 7, 512)    2048        average_pooling2d_2[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_87 (ReLU)                 (None, 7, 7, 512)    0           batch_normalization_87[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_88 (Conv2D)              (None, 7, 7, 128)    65664       re_lu_87[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_88 (BatchNo (None, 7, 7, 128)    512         conv2d_88[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_88 (ReLU)                 (None, 7, 7, 128)    0           batch_normalization_88[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_89 (Conv2D)              (None, 7, 7, 32)     36896       re_lu_88[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_42 (Concatenate)    (None, 7, 7, 544)    0           average_pooling2d_2[0][0]        \n",
            "                                                                 conv2d_89[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_89 (BatchNo (None, 7, 7, 544)    2176        concatenate_42[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_89 (ReLU)                 (None, 7, 7, 544)    0           batch_normalization_89[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_90 (Conv2D)              (None, 7, 7, 128)    69760       re_lu_89[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_90 (BatchNo (None, 7, 7, 128)    512         conv2d_90[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_90 (ReLU)                 (None, 7, 7, 128)    0           batch_normalization_90[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_91 (Conv2D)              (None, 7, 7, 32)     36896       re_lu_90[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_43 (Concatenate)    (None, 7, 7, 576)    0           concatenate_42[0][0]             \n",
            "                                                                 conv2d_91[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_91 (BatchNo (None, 7, 7, 576)    2304        concatenate_43[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_91 (ReLU)                 (None, 7, 7, 576)    0           batch_normalization_91[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_92 (Conv2D)              (None, 7, 7, 128)    73856       re_lu_91[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_92 (BatchNo (None, 7, 7, 128)    512         conv2d_92[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_92 (ReLU)                 (None, 7, 7, 128)    0           batch_normalization_92[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_93 (Conv2D)              (None, 7, 7, 32)     36896       re_lu_92[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_44 (Concatenate)    (None, 7, 7, 608)    0           concatenate_43[0][0]             \n",
            "                                                                 conv2d_93[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_93 (BatchNo (None, 7, 7, 608)    2432        concatenate_44[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_93 (ReLU)                 (None, 7, 7, 608)    0           batch_normalization_93[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_94 (Conv2D)              (None, 7, 7, 128)    77952       re_lu_93[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_94 (BatchNo (None, 7, 7, 128)    512         conv2d_94[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_94 (ReLU)                 (None, 7, 7, 128)    0           batch_normalization_94[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_95 (Conv2D)              (None, 7, 7, 32)     36896       re_lu_94[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_45 (Concatenate)    (None, 7, 7, 640)    0           concatenate_44[0][0]             \n",
            "                                                                 conv2d_95[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_95 (BatchNo (None, 7, 7, 640)    2560        concatenate_45[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_95 (ReLU)                 (None, 7, 7, 640)    0           batch_normalization_95[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_96 (Conv2D)              (None, 7, 7, 128)    82048       re_lu_95[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_96 (BatchNo (None, 7, 7, 128)    512         conv2d_96[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_96 (ReLU)                 (None, 7, 7, 128)    0           batch_normalization_96[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_97 (Conv2D)              (None, 7, 7, 32)     36896       re_lu_96[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_46 (Concatenate)    (None, 7, 7, 672)    0           concatenate_45[0][0]             \n",
            "                                                                 conv2d_97[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_97 (BatchNo (None, 7, 7, 672)    2688        concatenate_46[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_97 (ReLU)                 (None, 7, 7, 672)    0           batch_normalization_97[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_98 (Conv2D)              (None, 7, 7, 128)    86144       re_lu_97[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_98 (BatchNo (None, 7, 7, 128)    512         conv2d_98[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_98 (ReLU)                 (None, 7, 7, 128)    0           batch_normalization_98[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_99 (Conv2D)              (None, 7, 7, 32)     36896       re_lu_98[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_47 (Concatenate)    (None, 7, 7, 704)    0           concatenate_46[0][0]             \n",
            "                                                                 conv2d_99[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_99 (BatchNo (None, 7, 7, 704)    2816        concatenate_47[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_99 (ReLU)                 (None, 7, 7, 704)    0           batch_normalization_99[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_100 (Conv2D)             (None, 7, 7, 128)    90240       re_lu_99[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_100 (BatchN (None, 7, 7, 128)    512         conv2d_100[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_100 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_100[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_101 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_100[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_48 (Concatenate)    (None, 7, 7, 736)    0           concatenate_47[0][0]             \n",
            "                                                                 conv2d_101[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_101 (BatchN (None, 7, 7, 736)    2944        concatenate_48[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_101 (ReLU)                (None, 7, 7, 736)    0           batch_normalization_101[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_102 (Conv2D)             (None, 7, 7, 128)    94336       re_lu_101[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_102 (BatchN (None, 7, 7, 128)    512         conv2d_102[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_102 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_102[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_103 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_102[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_49 (Concatenate)    (None, 7, 7, 768)    0           concatenate_48[0][0]             \n",
            "                                                                 conv2d_103[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_103 (BatchN (None, 7, 7, 768)    3072        concatenate_49[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_103 (ReLU)                (None, 7, 7, 768)    0           batch_normalization_103[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_104 (Conv2D)             (None, 7, 7, 128)    98432       re_lu_103[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_104 (BatchN (None, 7, 7, 128)    512         conv2d_104[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_104 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_104[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_105 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_104[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_50 (Concatenate)    (None, 7, 7, 800)    0           concatenate_49[0][0]             \n",
            "                                                                 conv2d_105[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_105 (BatchN (None, 7, 7, 800)    3200        concatenate_50[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_105 (ReLU)                (None, 7, 7, 800)    0           batch_normalization_105[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_106 (Conv2D)             (None, 7, 7, 128)    102528      re_lu_105[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_106 (BatchN (None, 7, 7, 128)    512         conv2d_106[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_106 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_106[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_107 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_106[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_51 (Concatenate)    (None, 7, 7, 832)    0           concatenate_50[0][0]             \n",
            "                                                                 conv2d_107[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_107 (BatchN (None, 7, 7, 832)    3328        concatenate_51[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_107 (ReLU)                (None, 7, 7, 832)    0           batch_normalization_107[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_108 (Conv2D)             (None, 7, 7, 128)    106624      re_lu_107[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_108 (BatchN (None, 7, 7, 128)    512         conv2d_108[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_108 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_108[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_109 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_108[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_52 (Concatenate)    (None, 7, 7, 864)    0           concatenate_51[0][0]             \n",
            "                                                                 conv2d_109[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_109 (BatchN (None, 7, 7, 864)    3456        concatenate_52[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_109 (ReLU)                (None, 7, 7, 864)    0           batch_normalization_109[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_110 (Conv2D)             (None, 7, 7, 128)    110720      re_lu_109[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_110 (BatchN (None, 7, 7, 128)    512         conv2d_110[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_110 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_110[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_111 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_110[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_53 (Concatenate)    (None, 7, 7, 896)    0           concatenate_52[0][0]             \n",
            "                                                                 conv2d_111[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_111 (BatchN (None, 7, 7, 896)    3584        concatenate_53[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_111 (ReLU)                (None, 7, 7, 896)    0           batch_normalization_111[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_112 (Conv2D)             (None, 7, 7, 128)    114816      re_lu_111[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_112 (BatchN (None, 7, 7, 128)    512         conv2d_112[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_112 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_112[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_113 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_112[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_54 (Concatenate)    (None, 7, 7, 928)    0           concatenate_53[0][0]             \n",
            "                                                                 conv2d_113[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_113 (BatchN (None, 7, 7, 928)    3712        concatenate_54[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_113 (ReLU)                (None, 7, 7, 928)    0           batch_normalization_113[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_114 (Conv2D)             (None, 7, 7, 128)    118912      re_lu_113[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_114 (BatchN (None, 7, 7, 128)    512         conv2d_114[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_114 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_114[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_115 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_114[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_55 (Concatenate)    (None, 7, 7, 960)    0           concatenate_54[0][0]             \n",
            "                                                                 conv2d_115[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_115 (BatchN (None, 7, 7, 960)    3840        concatenate_55[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_115 (ReLU)                (None, 7, 7, 960)    0           batch_normalization_115[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_116 (Conv2D)             (None, 7, 7, 128)    123008      re_lu_115[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_116 (BatchN (None, 7, 7, 128)    512         conv2d_116[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_116 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_116[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_117 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_116[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_56 (Concatenate)    (None, 7, 7, 992)    0           concatenate_55[0][0]             \n",
            "                                                                 conv2d_117[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_117 (BatchN (None, 7, 7, 992)    3968        concatenate_56[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_117 (ReLU)                (None, 7, 7, 992)    0           batch_normalization_117[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_118 (Conv2D)             (None, 7, 7, 128)    127104      re_lu_117[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_118 (BatchN (None, 7, 7, 128)    512         conv2d_118[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_118 (ReLU)                (None, 7, 7, 128)    0           batch_normalization_118[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_119 (Conv2D)             (None, 7, 7, 32)     36896       re_lu_118[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_57 (Concatenate)    (None, 7, 7, 1024)   0           concatenate_56[0][0]             \n",
            "                                                                 conv2d_119[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling2d (Globa (None, 1024)         0           concatenate_57[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 1000)         1025000     global_average_pooling2d[0][0]   \n",
            "==================================================================================================\n",
            "Total params: 8,068,392\n",
            "Trainable params: 7,986,920\n",
            "Non-trainable params: 81,472\n",
            "__________________________________________________________________________________________________\n",
            "0.18874869346618653\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MuUxnO2BV9PG",
        "outputId": "d678473c-66e3-49e2-c98c-495534bf57a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def xception(input_shape, n_classes):\n",
        "  \n",
        "  def conv_bn(x, f, k, s=1, p='same'):\n",
        "    x = Conv2D(f, k, strides=s, padding=p, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def sep_bn(x, f, k, s=1, p='same'):\n",
        "    x = SeparableConv2D(f, k, strides=s, padding=p, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def entry_flow(x):\n",
        "    x = conv_bn(x, 32, 3, 2)\n",
        "    x = ReLU()(x)\n",
        "    x = conv_bn(x, 64, 3)\n",
        "    tensor = ReLU()(x)\n",
        "    \n",
        "    x = sep_bn(tensor, 128, 3)\n",
        "    x = ReLU()(x)\n",
        "    x = sep_bn(x, 128, 3)\n",
        "    x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "    \n",
        "    tensor = conv_bn(tensor, 128, 1, 2)\n",
        "    \n",
        "    x = Add()([tensor, x])\n",
        "    x = ReLU()(x)\n",
        "    x = sep_bn(x, 256, 3)\n",
        "    x = ReLU()(x)\n",
        "    x = sep_bn(x, 256, 3)\n",
        "    x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "    \n",
        "    tensor = conv_bn(tensor, 256, 1, 2)\n",
        "    \n",
        "    x = Add()([tensor, x])\n",
        "    x = ReLU()(x)\n",
        "    x = sep_bn(x, 728, 3)\n",
        "    x = ReLU()(x)\n",
        "    x = sep_bn(x, 728, 3)\n",
        "    x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "    \n",
        "    tensor = conv_bn(tensor, 728, 1, 2)\n",
        "    x = Add()([tensor, x])\n",
        "    \n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def middle_flow(tensor):\n",
        "    for _ in range(8):\n",
        "      x = ReLU()(tensor)\n",
        "      x = sep_bn(x, 728, 3)\n",
        "      x = ReLU()(x)\n",
        "      x = sep_bn(x, 728, 3)\n",
        "      x = ReLU()(x)\n",
        "      x = sep_bn(x, 728, 3)\n",
        "\n",
        "      tensor = Add()([tensor, x])\n",
        "    \n",
        "    return tensor\n",
        "  \n",
        "  \n",
        "  def exit_flow(tensor):\n",
        "    x = ReLU()(tensor)\n",
        "    x = sep_bn(x, 728, 3)\n",
        "    x = ReLU()(x)\n",
        "    x = sep_bn(x, 1024, 3)\n",
        "    x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "    \n",
        "    tensor = conv_bn(tensor, 1024, 1, 2)\n",
        "    \n",
        "    x = Add()([tensor, x])\n",
        "    x = sep_bn(x, 1536, 3)\n",
        "    x = ReLU()(x)\n",
        "    x = sep_bn(x, 2048, 3)\n",
        "    x = ReLU()(x)\n",
        "    x = GlobalAvgPool2D()(x)\n",
        "    x = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "    return x\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = entry_flow(input)\n",
        "  x = middle_flow(x)\n",
        "  output = exit_flow(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model\n",
        "\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = xception(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d (Conv2D)                 (None, 112, 112, 32) 864         input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization (BatchNorma (None, 112, 112, 32) 128         conv2d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu (ReLU)                    (None, 112, 112, 32) 0           batch_normalization[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 112, 112, 64) 18432       re_lu[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 112, 112, 64) 256         conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_1 (ReLU)                  (None, 112, 112, 64) 0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d (SeparableConv (None, 112, 112, 128 8768        re_lu_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 112, 112, 128 512         separable_conv2d[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_2 (ReLU)                  (None, 112, 112, 128 0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_1 (SeparableCo (None, 112, 112, 128 17536       re_lu_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 56, 56, 128)  8192        re_lu_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 112, 112, 128 512         separable_conv2d_1[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 56, 56, 128)  512         conv2d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D)    (None, 56, 56, 128)  0           batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "add (Add)                       (None, 56, 56, 128)  0           batch_normalization_4[0][0]      \n",
            "                                                                 max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_3 (ReLU)                  (None, 56, 56, 128)  0           add[0][0]                        \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_2 (SeparableCo (None, 56, 56, 256)  33920       re_lu_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 56, 56, 256)  1024        separable_conv2d_2[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_4 (ReLU)                  (None, 56, 56, 256)  0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_3 (SeparableCo (None, 56, 56, 256)  67840       re_lu_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 28, 28, 256)  32768       batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 56, 56, 256)  1024        separable_conv2d_3[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_7 (BatchNor (None, 28, 28, 256)  1024        conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 28, 28, 256)  0           batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "add_1 (Add)                     (None, 28, 28, 256)  0           batch_normalization_7[0][0]      \n",
            "                                                                 max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_5 (ReLU)                  (None, 28, 28, 256)  0           add_1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_4 (SeparableCo (None, 28, 28, 728)  188672      re_lu_5[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_8 (BatchNor (None, 28, 28, 728)  2912        separable_conv2d_4[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_6 (ReLU)                  (None, 28, 28, 728)  0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_5 (SeparableCo (None, 28, 28, 728)  536536      re_lu_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 14, 14, 728)  186368      batch_normalization_7[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_9 (BatchNor (None, 28, 28, 728)  2912        separable_conv2d_5[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_10 (BatchNo (None, 14, 14, 728)  2912        conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2D)  (None, 14, 14, 728)  0           batch_normalization_9[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "add_2 (Add)                     (None, 14, 14, 728)  0           batch_normalization_10[0][0]     \n",
            "                                                                 max_pooling2d_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_7 (ReLU)                  (None, 14, 14, 728)  0           add_2[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_6 (SeparableCo (None, 14, 14, 728)  536536      re_lu_7[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_11 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_6[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_8 (ReLU)                  (None, 14, 14, 728)  0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_7 (SeparableCo (None, 14, 14, 728)  536536      re_lu_8[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_12 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_7[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_9 (ReLU)                  (None, 14, 14, 728)  0           batch_normalization_12[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_8 (SeparableCo (None, 14, 14, 728)  536536      re_lu_9[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_13 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_8[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "add_3 (Add)                     (None, 14, 14, 728)  0           add_2[0][0]                      \n",
            "                                                                 batch_normalization_13[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_10 (ReLU)                 (None, 14, 14, 728)  0           add_3[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_9 (SeparableCo (None, 14, 14, 728)  536536      re_lu_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_14 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_9[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_11 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_10 (SeparableC (None, 14, 14, 728)  536536      re_lu_11[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_15 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_10[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_12 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_15[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_11 (SeparableC (None, 14, 14, 728)  536536      re_lu_12[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_16 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_11[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "add_4 (Add)                     (None, 14, 14, 728)  0           add_3[0][0]                      \n",
            "                                                                 batch_normalization_16[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_13 (ReLU)                 (None, 14, 14, 728)  0           add_4[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_12 (SeparableC (None, 14, 14, 728)  536536      re_lu_13[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_17 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_12[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_14 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_13 (SeparableC (None, 14, 14, 728)  536536      re_lu_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_18 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_13[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_15 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_18[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_14 (SeparableC (None, 14, 14, 728)  536536      re_lu_15[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_19 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_14[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "add_5 (Add)                     (None, 14, 14, 728)  0           add_4[0][0]                      \n",
            "                                                                 batch_normalization_19[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_16 (ReLU)                 (None, 14, 14, 728)  0           add_5[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_15 (SeparableC (None, 14, 14, 728)  536536      re_lu_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_20 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_15[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_17 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_16 (SeparableC (None, 14, 14, 728)  536536      re_lu_17[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_21 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_16[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_18 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_21[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_17 (SeparableC (None, 14, 14, 728)  536536      re_lu_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_22 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_17[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "add_6 (Add)                     (None, 14, 14, 728)  0           add_5[0][0]                      \n",
            "                                                                 batch_normalization_22[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_19 (ReLU)                 (None, 14, 14, 728)  0           add_6[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_18 (SeparableC (None, 14, 14, 728)  536536      re_lu_19[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_23 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_18[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_20 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_19 (SeparableC (None, 14, 14, 728)  536536      re_lu_20[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_24 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_19[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_21 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_24[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_20 (SeparableC (None, 14, 14, 728)  536536      re_lu_21[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_25 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_20[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "add_7 (Add)                     (None, 14, 14, 728)  0           add_6[0][0]                      \n",
            "                                                                 batch_normalization_25[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_22 (ReLU)                 (None, 14, 14, 728)  0           add_7[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_21 (SeparableC (None, 14, 14, 728)  536536      re_lu_22[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_26 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_21[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_23 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_26[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_22 (SeparableC (None, 14, 14, 728)  536536      re_lu_23[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_27 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_22[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_24 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_27[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_23 (SeparableC (None, 14, 14, 728)  536536      re_lu_24[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_28 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_23[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "add_8 (Add)                     (None, 14, 14, 728)  0           add_7[0][0]                      \n",
            "                                                                 batch_normalization_28[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_25 (ReLU)                 (None, 14, 14, 728)  0           add_8[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_24 (SeparableC (None, 14, 14, 728)  536536      re_lu_25[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_29 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_24[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_26 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_29[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_25 (SeparableC (None, 14, 14, 728)  536536      re_lu_26[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_30 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_25[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_27 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_30[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_26 (SeparableC (None, 14, 14, 728)  536536      re_lu_27[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_31 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_26[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "add_9 (Add)                     (None, 14, 14, 728)  0           add_8[0][0]                      \n",
            "                                                                 batch_normalization_31[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_28 (ReLU)                 (None, 14, 14, 728)  0           add_9[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_27 (SeparableC (None, 14, 14, 728)  536536      re_lu_28[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_32 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_27[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_29 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_32[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_28 (SeparableC (None, 14, 14, 728)  536536      re_lu_29[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_33 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_28[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_30 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_33[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_29 (SeparableC (None, 14, 14, 728)  536536      re_lu_30[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_34 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_29[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "add_10 (Add)                    (None, 14, 14, 728)  0           add_9[0][0]                      \n",
            "                                                                 batch_normalization_34[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_31 (ReLU)                 (None, 14, 14, 728)  0           add_10[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_30 (SeparableC (None, 14, 14, 728)  536536      re_lu_31[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_35 (BatchNo (None, 14, 14, 728)  2912        separable_conv2d_30[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_32 (ReLU)                 (None, 14, 14, 728)  0           batch_normalization_35[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_31 (SeparableC (None, 14, 14, 1024) 752024      re_lu_32[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 7, 7, 1024)   745472      add_10[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_36 (BatchNo (None, 14, 14, 1024) 4096        separable_conv2d_31[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_37 (BatchNo (None, 7, 7, 1024)   4096        conv2d_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_3 (MaxPooling2D)  (None, 7, 7, 1024)   0           batch_normalization_36[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "add_11 (Add)                    (None, 7, 7, 1024)   0           batch_normalization_37[0][0]     \n",
            "                                                                 max_pooling2d_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_32 (SeparableC (None, 7, 7, 1536)   1582080     add_11[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_38 (BatchNo (None, 7, 7, 1536)   6144        separable_conv2d_32[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_33 (ReLU)                 (None, 7, 7, 1536)   0           batch_normalization_38[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "separable_conv2d_33 (SeparableC (None, 7, 7, 2048)   3159552     re_lu_33[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_39 (BatchNo (None, 7, 7, 2048)   8192        separable_conv2d_33[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "re_lu_34 (ReLU)                 (None, 7, 7, 2048)   0           batch_normalization_39[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling2d (Globa (None, 2048)         0           re_lu_34[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 1000)         2049000     global_average_pooling2d[0][0]   \n",
            "==================================================================================================\n",
            "Total params: 22,910,480\n",
            "Trainable params: 22,855,952\n",
            "Non-trainable params: 54,528\n",
            "__________________________________________________________________________________________________\n",
            "0.21720128059387206\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rjB_zWyaWLD_"
      },
      "source": [
        "def unet(input_shape=(572, 572, 1), f=64, steps=4, n_classes=2):\n",
        "  \n",
        "  def downstream(x, f):\n",
        "    x = Conv2D(f, 3, activation='relu')(x)\n",
        "    d = Conv2D(f, 3, activation='relu')(x)\n",
        "    x = MaxPool2D(2, strides=2, padding='same')(d)\n",
        "    return d, x\n",
        "  \n",
        "  \n",
        "  def crop_merge(x, d):\n",
        "    _, xw, xh, _ = K.int_shape(x)\n",
        "    _, dw, dh, _ = K.int_shape(d)\n",
        "    mw, mh = (dw-xw)//2, (dh-xh)//2\n",
        "    \n",
        "    d = Lambda(lambda x: x[:, mw: dw-mw, mh: dh-mh, :])(d)\n",
        "    x = Concatenate()([d, x])\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  def upstream(x, f, d):\n",
        "    x = UpSampling2D()(x)\n",
        "    x = Conv2D(f, 2, padding='same')(x)\n",
        "    x = crop_merge(x, d)\n",
        "    x = Conv2D(f, 3, activation='relu')(x)\n",
        "    x = Conv2D(f, 3, activation='relu')(x)\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  x = input\n",
        "  \n",
        "  downsampled = []\n",
        "  for i in range(steps+1):\n",
        "    d, x = downstream(x, f*2**i)\n",
        "    downsampled.append(d)\n",
        "  x = downsampled.pop()\n",
        "  \n",
        "  for i in range(steps-1, -1, -1):\n",
        "    x = upstream(x, f*2**i, downsampled[i])\n",
        "  \n",
        "  output = Conv2D(n_classes, 1)(x)\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model\n",
        "\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = unet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k9ehEZMIWWie",
        "outputId": "bb594a56-8ae4-4f79-fa64-7b2ab8b94a6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def yolo(input_shape=(448, 448, 3), n_outputs=30):\n",
        "  activation = LeakyReLU(0.1)\n",
        "  \n",
        "  def conv_1_3(x, f1, f2, r=1):\n",
        "    for _ in range(r):\n",
        "      x = Conv2D(f1, 1, padding='same', activation=activation)(x)\n",
        "      x = Conv2D(f2, 3, padding='same', activation=activation)(x)\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(64, 7, strides=2, padding='same', activation=activation)(input)\n",
        "  x = MaxPool2D(2, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = Conv2D(192, 3, padding='same', activation=activation)(x)\n",
        "  x = MaxPool2D(2, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = conv_1_3(x, 128, 256)\n",
        "  x = conv_1_3(x, 256, 512)\n",
        "  x = MaxPool2D(2, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = conv_1_3(x, 256, 512, 4)\n",
        "  x = conv_1_3(x, 512, 1024)\n",
        "  x = MaxPool2D(2, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = conv_1_3(x, 512, 1024, 2)\n",
        "  x = Conv2D(1024, 3, padding='same', activation=activation)(x)\n",
        "  x = Conv2D(1024, 3, strides=2, padding='same', activation=activation)(x)\n",
        "  \n",
        "  x = Conv2D(1024, 3, padding='same', activation=activation)(x)\n",
        "  x = Conv2D(1024, 3, padding='same', activation=activation)(x)\n",
        "  \n",
        "  x = Dense(4096, activation=activation)(x)\n",
        "  output = Dense(n_outputs)(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = yolo(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "conv2d (Conv2D)              (None, 112, 112, 64)      9472      \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 56, 56, 64)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 56, 56, 192)       110784    \n",
            "_________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2 (None, 28, 28, 192)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 28, 28, 128)       24704     \n",
            "_________________________________________________________________\n",
            "conv2d_3 (Conv2D)            (None, 28, 28, 256)       295168    \n",
            "_________________________________________________________________\n",
            "conv2d_4 (Conv2D)            (None, 28, 28, 256)       65792     \n",
            "_________________________________________________________________\n",
            "conv2d_5 (Conv2D)            (None, 28, 28, 512)       1180160   \n",
            "_________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2 (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_6 (Conv2D)            (None, 14, 14, 256)       131328    \n",
            "_________________________________________________________________\n",
            "conv2d_7 (Conv2D)            (None, 14, 14, 512)       1180160   \n",
            "_________________________________________________________________\n",
            "conv2d_8 (Conv2D)            (None, 14, 14, 256)       131328    \n",
            "_________________________________________________________________\n",
            "conv2d_9 (Conv2D)            (None, 14, 14, 512)       1180160   \n",
            "_________________________________________________________________\n",
            "conv2d_10 (Conv2D)           (None, 14, 14, 256)       131328    \n",
            "_________________________________________________________________\n",
            "conv2d_11 (Conv2D)           (None, 14, 14, 512)       1180160   \n",
            "_________________________________________________________________\n",
            "conv2d_12 (Conv2D)           (None, 14, 14, 256)       131328    \n",
            "_________________________________________________________________\n",
            "conv2d_13 (Conv2D)           (None, 14, 14, 512)       1180160   \n",
            "_________________________________________________________________\n",
            "conv2d_14 (Conv2D)           (None, 14, 14, 512)       262656    \n",
            "_________________________________________________________________\n",
            "conv2d_15 (Conv2D)           (None, 14, 14, 1024)      4719616   \n",
            "_________________________________________________________________\n",
            "max_pooling2d_3 (MaxPooling2 (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_16 (Conv2D)           (None, 7, 7, 512)         524800    \n",
            "_________________________________________________________________\n",
            "conv2d_17 (Conv2D)           (None, 7, 7, 1024)        4719616   \n",
            "_________________________________________________________________\n",
            "conv2d_18 (Conv2D)           (None, 7, 7, 512)         524800    \n",
            "_________________________________________________________________\n",
            "conv2d_19 (Conv2D)           (None, 7, 7, 1024)        4719616   \n",
            "_________________________________________________________________\n",
            "conv2d_20 (Conv2D)           (None, 7, 7, 1024)        9438208   \n",
            "_________________________________________________________________\n",
            "conv2d_21 (Conv2D)           (None, 4, 4, 1024)        9438208   \n",
            "_________________________________________________________________\n",
            "conv2d_22 (Conv2D)           (None, 4, 4, 1024)        9438208   \n",
            "_________________________________________________________________\n",
            "conv2d_23 (Conv2D)           (None, 4, 4, 1024)        9438208   \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 4, 4, 4096)        4198400   \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 4, 4, 1000)        4097000   \n",
            "=================================================================\n",
            "Total params: 68,451,368\n",
            "Trainable params: 68,451,368\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "0.21383316516876222\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Zb79JoFWo3g"
      },
      "source": [
        "def refinenet(backbone_model, n_classes=21, tensor_names=('add_3', 'add_7', 'add_13', 'add_16')):\n",
        "  '''\n",
        "  backbone_model: (pretrained, non trainable) model used to extract the features (refinenet's inputs) \n",
        "  tensor_names: list of names of the backbone_model's tensors to be used as inputs (default: names from resnet)\n",
        "  '''\n",
        "  \n",
        "  filters = 512, 256, 256, 256\n",
        "  input = backbone_model.input\n",
        "  \n",
        "  def rcu(tensor, f, conv_shortcut=False):\n",
        "    x = ReLU()(tensor)\n",
        "    x = Conv2D(f, 3, padding='same')(x)\n",
        "    x = ReLU()(tensor)\n",
        "    x = Conv2D(f, 3, padding='same')(x)\n",
        "    \n",
        "    if conv_shortcut:\n",
        "      tensor = Conv2D(f, 3, padding='same')(tensor)\n",
        "      \n",
        "    output = Add()([tensor, x])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def fusion_block(x1, x2, f):\n",
        "    x1 = Conv2D(f, 3, padding='same')(x1)\n",
        "    \n",
        "    x2 = Conv2D(f, 3, padding='same')(x2)\n",
        "    x2 = UpSampling2D()(x2)\n",
        "    \n",
        "    output = Add()([x1, x2])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  \n",
        "  def chained_block(tensor, steps=2):\n",
        "    f = K.int_shape(tensor)[-1]\n",
        "    tensor = ReLU()(tensor)\n",
        "    \n",
        "    x = tensor\n",
        "    for _ in range(steps):\n",
        "      x = MaxPool2D(5, strides=1, padding='same')(x)\n",
        "      x = Conv2D(f, 3, padding='same')(x)\n",
        "      tensor = Add()([tensor, x])\n",
        "    \n",
        "    return tensor\n",
        "  \n",
        "  \n",
        "  def refine_block(x1, x2=None, f=256):\n",
        "    x = rcu(x1, f, True)\n",
        "    x = rcu(x, f)\n",
        "    \n",
        "    if x2 is not None:\n",
        "      x2 = rcu(x2, f, True)\n",
        "      x2 = rcu(x2, f)\n",
        "      x = fusion_block(x, x2, f)\n",
        "    \n",
        "    x = chained_block(x)\n",
        "    \n",
        "    output = rcu(x, f)\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  inputs = [layer.output for layer in backbone_model.layers if layer.name in tensor_names][::-1]\n",
        "  x = None\n",
        "  \n",
        "  for t, f in zip(inputs, filters):\n",
        "    x = refine_block(t, x, f)\n",
        "  \n",
        "  x = rcu(x, filters[-1])\n",
        "  x = rcu(x, filters[-1])\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model\n",
        "\n",
        "\n",
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = refinenet(resnet,input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "axXTbFvLW0mg"
      },
      "source": [
        "def vnet(input_shape=(128, 128, 64, 1), n_classes=2):\n",
        "  \n",
        "  def down_conv(x, filters):\n",
        "      x = Conv3D(filters=filters, kernel_size=2, strides=2, padding='same')(x)\n",
        "      x = PReLU()(x)\n",
        "      return x\n",
        "\n",
        "\n",
        "  def up_conv(x, filters):\n",
        "      x = Conv3DTranspose(filters=filters, kernel_size=2, strides=2, padding='same')(x)\n",
        "      x = PReLU()(x)\n",
        "      return x\n",
        "\n",
        "\n",
        "  def conv(x, filters):\n",
        "      x = Conv3D(filters=filters, kernel_size=5, strides=1, padding='same')(x)\n",
        "      x = PReLU()(x)\n",
        "      return x\n",
        "\n",
        "\n",
        "  def downstream_block(tensor, f1, f2, r):\n",
        "    x = tensor\n",
        "    for _ in range(r):\n",
        "      x = conv(x, filters=f1)\n",
        "    x = Add()([tensor, x])\n",
        "    forwards.append(x)\n",
        "    x = down_conv(x, filters=f2)\n",
        "    return x\n",
        "\n",
        "\n",
        "  n_filters = 16, 32, 64, 128, 256\n",
        "  repetitions = 1, 2, 3, 3, 3\n",
        "\n",
        "  forwards = []\n",
        "\n",
        "  # downstream (1st-4th block)\n",
        "  input = Input(input_shape)\n",
        "  tensor = input\n",
        "  for r, f1, f2 in zip(repetitions, n_filters, n_filters[1:]):\n",
        "    tensor = downstream_block(tensor, f1, f2, r)\n",
        "\n",
        "  # 5th block\n",
        "  x = tensor\n",
        "  for _ in range(repetitions[-1]):\n",
        "    x = conv(x, filters=n_filters[-1])\n",
        "  x = Add()([tensor, x])\n",
        "  tensor = up_conv(x, filters=n_filters[-1])\n",
        "\n",
        "\n",
        "  def upstream_block(tensor, f1, f2, r):\n",
        "    forward = forwards.pop()\n",
        "    x = Concatenate()([forward, tensor])\n",
        "    for _ in range(r):\n",
        "      x = conv(x, f1)\n",
        "    x = Add()([tensor, x])\n",
        "    x = up_conv(x, filters=f2)\n",
        "    return x\n",
        "\n",
        "\n",
        "  # upstream (6st-8th block)\n",
        "  for r, f1, f2 in zip(repetitions[-2:0:-1], n_filters[-1:0:-1], n_filters[-2:0:-1]):\n",
        "    tensor = upstream_block(tensor, f1, f2, r)\n",
        "\n",
        "  # 9th block\n",
        "  forward = forwards.pop()\n",
        "  x = Concatenate()([forward, tensor])\n",
        "  x = conv(x, filters=n_filters[1])\n",
        "  x = Add()([tensor, x])\n",
        "  output = Conv3D(filters=n_classes, kernel_size=1, activation='softmax')(x)\n",
        "\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model\n",
        "\n",
        "input_shape = 0,224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = vnet(input_shape, n_classes)\n",
        "model.summary()\n",
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}