Multi_Layer_Perceptron_Drill.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "hide_input": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    },
    "toc": {
      "colors": {
        "hover_highlight": "#DAA520",
        "running_highlight": "#FF0000",
        "selected_highlight": "#FFD700"
      },
      "moveMenuLeft": true,
      "nav_menu": {
        "height": "86px",
        "width": "252px"
      },
      "navigate_menu": true,
      "number_sections": true,
      "sideBar": true,
      "threshold": 4,
      "toc_cell": false,
      "toc_section_display": "block",
      "toc_window_display": false
    },
    "colab": {
      "name": "Copy of 4.3.3 Multi Layer Perceptron Drill.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cclarke411/Thinkful/blob/master/Multi_Layer_Perceptron_Drill.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "yOZlIG4p-Jkn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "nJLqzhG3-Jk0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "artworks = pd.read_csv('https://media.githubusercontent.com/media/MuseumofModernArt/collection/master/Artworks.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "scrolled": true,
        "id": "KsTQWvCY-Jk8",
        "colab_type": "code",
        "outputId": "63abc3a0-ea61-447b-ecad-66697db748da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "artworks.columns"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['Artist', 'Nationality', 'Gender', 'Date', 'Department', 'DateAcquired',\n",
              "       'URL', 'ThumbnailURL', 'Height (cm)', 'Width (cm)'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "vV1SE_Gi-JlQ",
        "colab_type": "text"
      },
      "source": [
        "We'll also do a bit of data processing and cleaning, selecting columns of interest and converting URL's to booleans indicating whether they are present."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "U8jSTYed-JlT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Select Columns.\n",
        "artworks = artworks[['Artist', 'Nationality', 'Gender', 'Date', 'Department',\n",
        "                    'DateAcquired', 'URL', 'ThumbnailURL', 'Height (cm)', 'Width (cm)']]\n",
        "\n",
        "# Convert URL's to booleans.\n",
        "artworks['URL'] = artworks['URL'].notnull()\n",
        "artworks['ThumbnailURL'] = artworks['ThumbnailURL'].notnull()\n",
        "\n",
        "# Drop films and some other tricky rows.\n",
        "artworks = artworks[artworks['Department']!='Film']\n",
        "artworks = artworks[artworks['Department']!='Media and Performance Art']\n",
        "artworks = artworks[artworks['Department']!='Fluxus Collection']\n",
        "\n",
        "# Drop missing data.\n",
        "artworks = artworks.dropna()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "XJg-Dr2m-Jlg",
        "colab_type": "code",
        "outputId": "a8243f73-a2af-4df5-a0fa-4f23d16a6c60",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "artworks.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Artist</th>\n",
              "      <th>Nationality</th>\n",
              "      <th>Gender</th>\n",
              "      <th>Date</th>\n",
              "      <th>Department</th>\n",
              "      <th>DateAcquired</th>\n",
              "      <th>URL</th>\n",
              "      <th>ThumbnailURL</th>\n",
              "      <th>Height (cm)</th>\n",
              "      <th>Width (cm)</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Otto Wagner</td>\n",
              "      <td>(Austrian)</td>\n",
              "      <td>(Male)</td>\n",
              "      <td>1896</td>\n",
              "      <td>Architecture &amp; Design</td>\n",
              "      <td>1996-04-09</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>48.6000</td>\n",
              "      <td>168.9000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Christian de Portzamparc</td>\n",
              "      <td>(French)</td>\n",
              "      <td>(Male)</td>\n",
              "      <td>1987</td>\n",
              "      <td>Architecture &amp; Design</td>\n",
              "      <td>1995-01-17</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>40.6401</td>\n",
              "      <td>29.8451</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Emil Hoppe</td>\n",
              "      <td>(Austrian)</td>\n",
              "      <td>(Male)</td>\n",
              "      <td>1903</td>\n",
              "      <td>Architecture &amp; Design</td>\n",
              "      <td>1997-01-15</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>34.3000</td>\n",
              "      <td>31.8000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Bernard Tschumi</td>\n",
              "      <td>()</td>\n",
              "      <td>(Male)</td>\n",
              "      <td>1980</td>\n",
              "      <td>Architecture &amp; Design</td>\n",
              "      <td>1995-01-17</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>50.8000</td>\n",
              "      <td>50.8000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Emil Hoppe</td>\n",
              "      <td>(Austrian)</td>\n",
              "      <td>(Male)</td>\n",
              "      <td>1903</td>\n",
              "      <td>Architecture &amp; Design</td>\n",
              "      <td>1997-01-15</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>38.4000</td>\n",
              "      <td>19.1000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                     Artist Nationality  ... Height (cm) Width (cm)\n",
              "0               Otto Wagner  (Austrian)  ...     48.6000   168.9000\n",
              "1  Christian de Portzamparc    (French)  ...     40.6401    29.8451\n",
              "2                Emil Hoppe  (Austrian)  ...     34.3000    31.8000\n",
              "3           Bernard Tschumi          ()  ...     50.8000    50.8000\n",
              "4                Emil Hoppe  (Austrian)  ...     38.4000    19.1000\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "ohclJ9mh-Jlt",
        "colab_type": "code",
        "outputId": "eeb85a28-01dc-4a2f-c618-67c79007563d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# Get data types.\n",
        "artworks.dtypes"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Artist           object\n",
              "Nationality      object\n",
              "Gender           object\n",
              "Date             object\n",
              "Department       object\n",
              "DateAcquired     object\n",
              "URL                bool\n",
              "ThumbnailURL       bool\n",
              "Height (cm)     float64\n",
              "Width (cm)      float64\n",
              "dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "b_4JFSpp-Jl2",
        "colab_type": "code",
        "outputId": "712593cc-4a1b-475c-866d-52521765475f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "artworks['DateAcquired'] = pd.to_datetime(artworks.DateAcquired)\n",
        "artworks['YearAcquired'] = artworks.DateAcquired.dt.year\n",
        "artworks['YearAcquired'].dtype"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dtype('int64')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "0Zq0wndM-JmB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Remove multiple nationalities, genders, and artists.\n",
        "artworks.loc[artworks['Gender'].str.contains('\\) \\('), 'Gender'] = '\\(multiple_persons\\)'\n",
        "artworks.loc[artworks['Nationality'].str.contains('\\) \\('), 'Nationality'] = '\\(multiple_nationalities\\)'\n",
        "artworks.loc[artworks['Artist'].str.contains(','), 'Artist'] = 'Multiple_Artists'\n",
        "\n",
        "# Convert dates to start date, cutting down number of distinct examples.\n",
        "artworks['Date'] = pd.Series(artworks.Date.str.extract(\n",
        "    '([0-9]{4})', expand=False))[:-1]\n",
        "\n",
        "# Final column drops and NA drop.\n",
        "X = artworks.drop(['Department', 'DateAcquired', 'Artist', 'Nationality', 'Date'], 1)\n",
        "\n",
        "# Create dummies separately.\n",
        "artists = pd.get_dummies(artworks.Artist)\n",
        "nationalities = pd.get_dummies(artworks.Nationality)\n",
        "dates = pd.get_dummies(artworks.Date)\n",
        "\n",
        "# Concat with other variables, but artists slows this wayyyyy down so we'll keep it out for now\n",
        "X = pd.get_dummies(X, sparse=True)\n",
        "X = pd.concat([X, nationalities, dates], axis=1)\n",
        "\n",
        "Y = artworks.Department"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "jHweXY-l-JmK",
        "colab_type": "code",
        "outputId": "68b82029-d6f3-4469-e028-b6ec637676b9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "# Alright! We've done our prep, let's build the model.\n",
        "# Neural networks are hugely computationally intensive.\n",
        "# This may take several minutes to run.\n",
        "\n",
        "# Import the model.\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "\n",
        "# Establish and fit the model, with a single, 1000 perceptron layer.\n",
        "mlp = MLPClassifier(hidden_layer_sizes=(10,))\n",
        "mlp.fit(X, Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
              "              beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
              "              hidden_layer_sizes=(10,), learning_rate='constant',\n",
              "              learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
              "              n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
              "              random_state=None, shuffle=True, solver='adam', tol=0.0001,\n",
              "              validation_fraction=0.1, verbose=False, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lgz9aWdAOl_7",
        "colab_type": "code",
        "outputId": "808a2883-1a45-4815-88aa-4541335ede39",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "X.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(108418, 319)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rWKNAeL--JmW",
        "colab_type": "code",
        "outputId": "76b85284-0039-4ea4-d1a1-361da8b947a2",
        "colab": {}
      },
      "source": [
        "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, stratify=Y, test_size=.50)\n",
        "mlp = MLPClassifier(hidden_layer_sizes=(100,), verbose=True)\n",
        "mlp.fit(X_train, Y_train)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Iteration 1, loss = 8.53218149\n",
            "Iteration 2, loss = inf\n",
            "Iteration 3, loss = 4.10435227\n",
            "Iteration 4, loss = 2.11884320\n",
            "Iteration 5, loss = 2.98945200\n",
            "Iteration 6, loss = 2.92412752\n",
            "Iteration 7, loss = 2.19561599\n",
            "Iteration 8, loss = 1.71580036\n",
            "Iteration 9, loss = 2.40673376\n",
            "Iteration 10, loss = 2.30559583\n",
            "Iteration 11, loss = 1.30362327\n",
            "Iteration 12, loss = 2.19639829\n",
            "Iteration 13, loss = 1.58683728\n",
            "Iteration 14, loss = 1.49535096\n",
            "Iteration 15, loss = 1.35862250\n",
            "Iteration 16, loss = 1.64602528\n",
            "Iteration 17, loss = 1.18093830\n",
            "Iteration 18, loss = 1.64699567\n",
            "Iteration 19, loss = 1.82761999\n",
            "Iteration 20, loss = 1.09532386\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "6nwbenws-Jmk",
        "colab_type": "code",
        "outputId": "e6af783d-756f-4b45-e8bd-71d8e9519817",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "mlp.score(X, Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7433175302993967"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "wdXtsEIf-Jmv",
        "colab_type": "code",
        "outputId": "b438d408-4f8a-4260-da28-8851898893dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "Y.value_counts()/len(Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Drawings & Prints        0.622507\n",
              "Photography              0.226466\n",
              "Architecture & Design    0.112970\n",
              "Painting & Sculpture     0.033684\n",
              "Media and Performance    0.004372\n",
              "Name: Department, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "run_control": {
          "frozen": false,
          "read_only": false
        },
        "id": "yqfXdqrX-Jm8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import cross_val_score\n",
        "cross_val_score(mlp, Xtest, Ytest, cv=2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZGRZKtX-Jnq",
        "colab_type": "text"
      },
      "source": [
        "## Drill: Playing with layers\n",
        "\n",
        "Now it's your turn. Using the space below, experiment with different hidden layer structures. You can try this on a subset of the data to improve runtime. See how things vary. See what seems to matter the most. Feel free to manipulate other parameters as well. It may also be beneficial to do some real feature selection work..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SzkyGxuE-Jns",
        "colab_type": "code",
        "outputId": "bbb7aed5-ccaa-4d9d-819e-4a671fdda543",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "# Your code here. Experiment with hidden layers to build your own model.\n",
        "\n",
        "mlp = MLPClassifier(hidden_layer_sizes=(20,))\n",
        "mlp.fit(X, Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
              "              beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
              "              hidden_layer_sizes=(20,), learning_rate='constant',\n",
              "              learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
              "              n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
              "              random_state=None, shuffle=True, solver='adam', tol=0.0001,\n",
              "              validation_fraction=0.1, verbose=False, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d1MAoIXV-Jn3",
        "colab_type": "code",
        "outputId": "ce26308d-0c63-46f3-969f-61b29a7853b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "mlp.score(X, Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7771772214945858"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eou0bVNa-JoE",
        "colab_type": "code",
        "outputId": "5ba4d486-20f3-47e7-e647-4fafcc8453c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "Y.value_counts()/len(Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Drawings & Prints        0.622507\n",
              "Photography              0.226466\n",
              "Architecture & Design    0.112970\n",
              "Painting & Sculpture     0.033684\n",
              "Media and Performance    0.004372\n",
              "Name: Department, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QiLrcDmY-JoO",
        "colab_type": "code",
        "outputId": "c8c6071a-44b8-441d-c214-679af2885322",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "mlp = MLPClassifier(hidden_layer_sizes=(20,),activation='relu',max_iter=100)\n",
        "mlp.fit(X, Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
              "              beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
              "              hidden_layer_sizes=(20,), learning_rate='constant',\n",
              "              learning_rate_init=0.001, max_iter=100, momentum=0.9,\n",
              "              n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
              "              random_state=None, shuffle=True, solver='adam', tol=0.0001,\n",
              "              validation_fraction=0.1, verbose=False, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Is6E7frhRqGR",
        "colab_type": "code",
        "outputId": "238e955a-2f23-46b6-9177-7377268b9583",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "mlp.score(X, Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7346842775184932"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ppCgCUp_RrFU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}