pandas_google_colab_demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMNrMvV7nFa63VTDftwDcTg",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tongakuot/python_tutorials/blob/main/pandas_google_colab_demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Cleaning and Transforming Data with `pandas`"
      ],
      "metadata": {
        "id": "fdqumYsyGRu0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loading Libraries"
      ],
      "metadata": {
        "id": "u7Ye-72vGs8u"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GZlvJ8v2Fzwz",
        "outputId": "7e79f09a-aa0d-4c36-aef0-fd7861680ee6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "#!pip install pyjanitor\n",
        "from janitor import clean_names"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Importing the Dataset"
      ],
      "metadata": {
        "id": "hOvrm7EYHW1d"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 94
        },
        "id": "uy8AcoUVG-db",
        "outputId": "1c9c3340-6ce4-4e26-8d8e-8756f19d19e1"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-56a85183-7c8c-42d6-80b7-bfe25b215e2f\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-56a85183-7c8c-42d6-80b7-bfe25b215e2f\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "census_raw = pd.read_csv('/content/ss_2008_census_data_raw.csv')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0u9tQ5b-IArw",
        "outputId": "633f1e0d-d225-419c-8076-3a4b87785b85"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Inspect the first few rows\n",
        "census_raw.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262
        },
        "id": "rEt64SRnIK6B",
        "outputId": "94698d9c-0591-491b-fee7-72d9354629df"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  Region Region Name Region - RegionId Variable               Variable Name  \\\n",
              "0  KN.A2  Upper Nile             SS-NU    KN.B2  Population, Total (Number)   \n",
              "1  KN.A2  Upper Nile             SS-NU    KN.B2  Population, Total (Number)   \n",
              "2  KN.A2  Upper Nile             SS-NU    KN.B2  Population, Total (Number)   \n",
              "3  KN.A2  Upper Nile             SS-NU    KN.B2  Population, Total (Number)   \n",
              "4  KN.A2  Upper Nile             SS-NU    KN.B2  Population, Total (Number)   \n",
              "\n",
              "     Age  Age Name  Scale    Units      2008  \n",
              "0  KN.C1     Total  units  Persons  964353.0  \n",
              "1  KN.C2    0 to 4  units  Persons  150872.0  \n",
              "2  KN.C3    5 to 9  units  Persons  151467.0  \n",
              "3  KN.C4  10 to 14  units  Persons  126140.0  \n",
              "4  KN.C5  15 to 19  units  Persons  103804.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7ee6c145-83cf-4871-8e9d-574b96e4b44b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Region</th>\n",
              "      <th>Region Name</th>\n",
              "      <th>Region - RegionId</th>\n",
              "      <th>Variable</th>\n",
              "      <th>Variable Name</th>\n",
              "      <th>Age</th>\n",
              "      <th>Age Name</th>\n",
              "      <th>Scale</th>\n",
              "      <th>Units</th>\n",
              "      <th>2008</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>KN.A2</td>\n",
              "      <td>Upper Nile</td>\n",
              "      <td>SS-NU</td>\n",
              "      <td>KN.B2</td>\n",
              "      <td>Population, Total (Number)</td>\n",
              "      <td>KN.C1</td>\n",
              "      <td>Total</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>964353.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>KN.A2</td>\n",
              "      <td>Upper Nile</td>\n",
              "      <td>SS-NU</td>\n",
              "      <td>KN.B2</td>\n",
              "      <td>Population, Total (Number)</td>\n",
              "      <td>KN.C2</td>\n",
              "      <td>0 to 4</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>150872.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>KN.A2</td>\n",
              "      <td>Upper Nile</td>\n",
              "      <td>SS-NU</td>\n",
              "      <td>KN.B2</td>\n",
              "      <td>Population, Total (Number)</td>\n",
              "      <td>KN.C3</td>\n",
              "      <td>5 to 9</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>151467.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>KN.A2</td>\n",
              "      <td>Upper Nile</td>\n",
              "      <td>SS-NU</td>\n",
              "      <td>KN.B2</td>\n",
              "      <td>Population, Total (Number)</td>\n",
              "      <td>KN.C4</td>\n",
              "      <td>10 to 14</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>126140.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>KN.A2</td>\n",
              "      <td>Upper Nile</td>\n",
              "      <td>SS-NU</td>\n",
              "      <td>KN.B2</td>\n",
              "      <td>Population, Total (Number)</td>\n",
              "      <td>KN.C5</td>\n",
              "      <td>15 to 19</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>103804.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7ee6c145-83cf-4871-8e9d-574b96e4b44b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7ee6c145-83cf-4871-8e9d-574b96e4b44b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7ee6c145-83cf-4871-8e9d-574b96e4b44b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Inspect the last 10 rows\n",
        "census_raw.tail(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "KiBiEb9eIQ9I",
        "outputId": "e39bbe9b-7ab9-4b61-f4b1-1706605b75f3"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Region                                        Region Name  \\\n",
              "443         KN.A11                                  Eastern Equatoria   \n",
              "444         KN.A11                                  Eastern Equatoria   \n",
              "445         KN.A11                                  Eastern Equatoria   \n",
              "446         KN.A11                                  Eastern Equatoria   \n",
              "447         KN.A11                                  Eastern Equatoria   \n",
              "448         KN.A11                                  Eastern Equatoria   \n",
              "449         KN.A11                                  Eastern Equatoria   \n",
              "450            NaN                                                NaN   \n",
              "451        Source:         National Bureau of Statistics, South Sudan   \n",
              "452  Download URL:  http://southsudan.opendataforafrica.org/fvjqdp...   \n",
              "\n",
              "    Region - RegionId Variable                Variable Name     Age  Age Name  \\\n",
              "443             SS-EE    KN.B8  Population, Female (Number)   KN.C9  35 to 39   \n",
              "444             SS-EE    KN.B8  Population, Female (Number)  KN.C10  40 to 44   \n",
              "445             SS-EE    KN.B8  Population, Female (Number)  KN.C11  45 to 49   \n",
              "446             SS-EE    KN.B8  Population, Female (Number)  KN.C12  50 to 54   \n",
              "447             SS-EE    KN.B8  Population, Female (Number)  KN.C13  55 to 59   \n",
              "448             SS-EE    KN.B8  Population, Female (Number)  KN.C14  60 to 64   \n",
              "449             SS-EE    KN.B8  Population, Female (Number)  KN.C22       65+   \n",
              "450               NaN      NaN                          NaN     NaN       NaN   \n",
              "451               NaN      NaN                          NaN     NaN       NaN   \n",
              "452               NaN      NaN                          NaN     NaN       NaN   \n",
              "\n",
              "     Scale    Units     2008  \n",
              "443  units  Persons  25808.0  \n",
              "444  units  Persons  17585.0  \n",
              "445  units  Persons  13727.0  \n",
              "446  units  Persons   9482.0  \n",
              "447  units  Persons   5740.0  \n",
              "448  units  Persons   5274.0  \n",
              "449  units  Persons   8637.0  \n",
              "450    NaN      NaN      NaN  \n",
              "451    NaN      NaN      NaN  \n",
              "452    NaN      NaN      NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-abbb6339-3464-4ff7-9b6e-c0caa7b469d6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Region</th>\n",
              "      <th>Region Name</th>\n",
              "      <th>Region - RegionId</th>\n",
              "      <th>Variable</th>\n",
              "      <th>Variable Name</th>\n",
              "      <th>Age</th>\n",
              "      <th>Age Name</th>\n",
              "      <th>Scale</th>\n",
              "      <th>Units</th>\n",
              "      <th>2008</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>443</th>\n",
              "      <td>KN.A11</td>\n",
              "      <td>Eastern Equatoria</td>\n",
              "      <td>SS-EE</td>\n",
              "      <td>KN.B8</td>\n",
              "      <td>Population, Female (Number)</td>\n",
              "      <td>KN.C9</td>\n",
              "      <td>35 to 39</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>25808.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>444</th>\n",
              "      <td>KN.A11</td>\n",
              "      <td>Eastern Equatoria</td>\n",
              "      <td>SS-EE</td>\n",
              "      <td>KN.B8</td>\n",
              "      <td>Population, Female (Number)</td>\n",
              "      <td>KN.C10</td>\n",
              "      <td>40 to 44</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>17585.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>445</th>\n",
              "      <td>KN.A11</td>\n",
              "      <td>Eastern Equatoria</td>\n",
              "      <td>SS-EE</td>\n",
              "      <td>KN.B8</td>\n",
              "      <td>Population, Female (Number)</td>\n",
              "      <td>KN.C11</td>\n",
              "      <td>45 to 49</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>13727.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>446</th>\n",
              "      <td>KN.A11</td>\n",
              "      <td>Eastern Equatoria</td>\n",
              "      <td>SS-EE</td>\n",
              "      <td>KN.B8</td>\n",
              "      <td>Population, Female (Number)</td>\n",
              "      <td>KN.C12</td>\n",
              "      <td>50 to 54</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>9482.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>447</th>\n",
              "      <td>KN.A11</td>\n",
              "      <td>Eastern Equatoria</td>\n",
              "      <td>SS-EE</td>\n",
              "      <td>KN.B8</td>\n",
              "      <td>Population, Female (Number)</td>\n",
              "      <td>KN.C13</td>\n",
              "      <td>55 to 59</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>5740.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>448</th>\n",
              "      <td>KN.A11</td>\n",
              "      <td>Eastern Equatoria</td>\n",
              "      <td>SS-EE</td>\n",
              "      <td>KN.B8</td>\n",
              "      <td>Population, Female (Number)</td>\n",
              "      <td>KN.C14</td>\n",
              "      <td>60 to 64</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>5274.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>449</th>\n",
              "      <td>KN.A11</td>\n",
              "      <td>Eastern Equatoria</td>\n",
              "      <td>SS-EE</td>\n",
              "      <td>KN.B8</td>\n",
              "      <td>Population, Female (Number)</td>\n",
              "      <td>KN.C22</td>\n",
              "      <td>65+</td>\n",
              "      <td>units</td>\n",
              "      <td>Persons</td>\n",
              "      <td>8637.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>450</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>451</th>\n",
              "      <td>Source:</td>\n",
              "      <td>National Bureau of Statistics, South Sudan</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>452</th>\n",
              "      <td>Download URL:</td>\n",
              "      <td>http://southsudan.opendataforafrica.org/fvjqdp...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-abbb6339-3464-4ff7-9b6e-c0caa7b469d6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-abbb6339-3464-4ff7-9b6e-c0caa7b469d6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-abbb6339-3464-4ff7-9b6e-c0caa7b469d6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Dimensions\n",
        "census_raw.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9xukzpyZIjzV",
        "outputId": "ed6ec0aa-6620-4640-d164-82a32b1dc204"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(453, 10)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Selecting desired columns"
      ],
      "metadata": {
        "id": "Qqp3MiO5I4-F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "census_raw.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g_B9zmEXIvqX",
        "outputId": "7df79deb-5816-4c2d-f04b-19a1ec5b2a2b"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['Region', 'Region Name', 'Region - RegionId', 'Variable',\n",
              "       'Variable Name', 'Age', 'Age Name', 'Scale', 'Units', '2008'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Checking for missing values\n",
        "#census_raw.isna().sum()\n",
        "\n",
        "census_raw[census_raw['2008'].isna()]\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        },
        "id": "P384iaDyJwRx",
        "outputId": "64387eab-79fa-4af4-8fd5-d01cd13c8d72"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Region                                        Region Name  \\\n",
              "450            NaN                                                NaN   \n",
              "451        Source:         National Bureau of Statistics, South Sudan   \n",
              "452  Download URL:  http://southsudan.opendataforafrica.org/fvjqdp...   \n",
              "\n",
              "    Region - RegionId Variable Variable Name  Age Age Name Scale Units  2008  \n",
              "450               NaN      NaN           NaN  NaN      NaN   NaN   NaN   NaN  \n",
              "451               NaN      NaN           NaN  NaN      NaN   NaN   NaN   NaN  \n",
              "452               NaN      NaN           NaN  NaN      NaN   NaN   NaN   NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3042b21c-b959-4833-b77e-bdcabf3a3943\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Region</th>\n",
              "      <th>Region Name</th>\n",
              "      <th>Region - RegionId</th>\n",
              "      <th>Variable</th>\n",
              "      <th>Variable Name</th>\n",
              "      <th>Age</th>\n",
              "      <th>Age Name</th>\n",
              "      <th>Scale</th>\n",
              "      <th>Units</th>\n",
              "      <th>2008</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>450</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>451</th>\n",
              "      <td>Source:</td>\n",
              "      <td>National Bureau of Statistics, South Sudan</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>452</th>\n",
              "      <td>Download URL:</td>\n",
              "      <td>http://southsudan.opendataforafrica.org/fvjqdp...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3042b21c-b959-4833-b77e-bdcabf3a3943')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3042b21c-b959-4833-b77e-bdcabf3a3943 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3042b21c-b959-4833-b77e-bdcabf3a3943');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Checking for data types\n",
        "census_raw.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tf4sN2-GKu_e",
        "outputId": "d629697c-3da3-4159-a39c-04d3bf4dba7a"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 453 entries, 0 to 452\n",
            "Data columns (total 10 columns):\n",
            " #   Column             Non-Null Count  Dtype  \n",
            "---  ------             --------------  -----  \n",
            " 0   Region             452 non-null    object \n",
            " 1   Region Name        452 non-null    object \n",
            " 2   Region - RegionId  450 non-null    object \n",
            " 3   Variable           450 non-null    object \n",
            " 4   Variable Name      450 non-null    object \n",
            " 5   Age                450 non-null    object \n",
            " 6   Age Name           450 non-null    object \n",
            " 7   Scale              450 non-null    object \n",
            " 8   Units              450 non-null    object \n",
            " 9   2008               450 non-null    float64\n",
            "dtypes: float64(1), object(9)\n",
            "memory usage: 35.5+ KB\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cols = ['Region Name','Variable Name', 'Age Name', '2008']\n",
        "\n",
        "age_maps = {'0 to 4':'0-14', '5 to 9':'0-14', \n",
        "            '10 to 14':'0-14', '15 to 19':'15-29', \n",
        "            '20 to 24':'15-29', '25 to 29':'15-29',\n",
        "            '30 to 34':'30-49', '35 to 39':'30-49', \n",
        "            '40 to 44':'30-49', '45 to 49':'30-49', \n",
        "            '50 to 54':'50-64', '55 to 59':'50-64', \n",
        "            '60 to 64':'50-64', '65+':'65+'}\n",
        "\n",
        "# Subset the data\n",
        "census = (\n",
        "    census_raw\n",
        "    [cols]\n",
        "    .clean_names()\n",
        "    .dropna()\n",
        "    .rename(columns={'region_name':'state', \n",
        "                    'variable_name':'gender',\n",
        "                    'age_name':'age_category',\n",
        "                    '2008':'population'}\n",
        "          )\n",
        "    .assign(\n",
        "        population = lambda x: x['population'].astype('int'),\n",
        "        gender = lambda x: x['gender'].str.split(' ').str[1]\n",
        "    )\n",
        "    .query('gender != \"Total\" & age_category != \"Total\"')\n",
        "    .assign(\n",
        "        age_category = lambda x: x['age_category'].map(age_maps)\n",
        "    )\n",
        "    .groupby(['state', 'gender', 'age_category'])\n",
        "    ['population']\n",
        "    .sum(numeric_only=True)\n",
        "\n",
        ")\n",
        "\n",
        "census"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9YkiRdA1I_4O",
        "outputId": "d2d5c9c1-8e3a-421f-d341-ebb87c68d98c"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "state              gender  age_category\n",
              "Central Equatoria  Female  0-14            221216\n",
              "                           15-29           166887\n",
              "                           30-49           101676\n",
              "                           50-64            23460\n",
              "                           65+               8596\n",
              "                                            ...  \n",
              "Western Equatoria  Male    0-14            125766\n",
              "                           15-29            93082\n",
              "                           30-49            68530\n",
              "                           50-64            19524\n",
              "                           65+              11541\n",
              "Name: population, Length: 100, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "census.age_category.unique()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DrAIhg3xKjGP",
        "outputId": "744ef725-7eb0-4163-e72f-c9d996ba0902"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['0-14', '15-29', '30-49', '50-64', '65+'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Write a function\n",
        "col_maps ={'region_name':'state', \n",
        "                    'variable_name':'gender',\n",
        "                    'age_name':'age_category',\n",
        "                    '2008':'population'}\n",
        "\n",
        "age_maps = {'0 to 4':'0-14', '5 to 9':'0-14', \n",
        "              '10 to 14':'0-14', '15 to 19':'15-29', \n",
        "              '20 to 24':'15-29', '25 to 29':'15-29',\n",
        "              '30 to 34':'30-49', '35 to 39':'30-49', \n",
        "              '40 to 44':'30-49', '45 to 49':'30-49', \n",
        "              '50 to 54':'50-64', '55 to 59':'50-64', \n",
        "              '60 to 64':'50-64', '65+':'65+'}\n",
        "\n",
        "def tweak_census_data(df):\n",
        "  \n",
        "  cols = ['Region Name','Variable Name', 'Age Name', '2008']\n",
        "\n",
        "  # Subset the data\n",
        "  census = (\n",
        "      df\n",
        "      [cols]\n",
        "      .clean_names()\n",
        "      .dropna()\n",
        "      .rename(columns=col_maps)\n",
        "      .assign(\n",
        "          population = lambda x: x['population'].astype('int'),\n",
        "          gender = lambda x: x['gender'].str.split(' ').str[1]\n",
        "      )\n",
        "      .query('gender != \"Total\" & age_category != \"Total\"')\n",
        "      .assign(\n",
        "          age_category = lambda x: x['age_category'].map(age_maps)\n",
        "      )\n",
        "      .groupby(['state', 'gender', 'age_category'])\n",
        "      ['population']\n",
        "      .sum(numeric_only=True)\n",
        "  \n",
        "  )\n",
        "\n",
        "  return census"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mU9sDlaKNPiI",
        "outputId": "305870a8-705c-48d5-bfe7-6fbcf81874d7"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Testing the function\n",
        "df = pd.read_csv('/content/ss_2008_census_data_raw.csv')\n",
        "tweak_census_data(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        },
        "id": "b1rCaIUBQ71m",
        "outputId": "bc2c98e6-1bbb-4f55-9032-ce16ee450974"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                       population\n",
              "state             gender age_category            \n",
              "Central Equatoria Female 0-14              221216\n",
              "                         15-29             166887\n",
              "                         30-49             101676\n",
              "                         50-64              23460\n",
              "                         65+                 8596\n",
              "...                                           ...\n",
              "Western Equatoria Male   0-14              125766\n",
              "                         15-29              93082\n",
              "                         30-49              68530\n",
              "                         50-64              19524\n",
              "                         65+                11541\n",
              "\n",
              "[100 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-35edfb28-3eb9-4d2e-8f80-0f162d5eaad4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>population</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>state</th>\n",
              "      <th>gender</th>\n",
              "      <th>age_category</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">Central Equatoria</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">Female</th>\n",
              "      <th>0-14</th>\n",
              "      <td>221216</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15-29</th>\n",
              "      <td>166887</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30-49</th>\n",
              "      <td>101676</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50-64</th>\n",
              "      <td>23460</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65+</th>\n",
              "      <td>8596</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">Western Equatoria</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">Male</th>\n",
              "      <th>0-14</th>\n",
              "      <td>125766</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15-29</th>\n",
              "      <td>93082</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30-49</th>\n",
              "      <td>68530</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50-64</th>\n",
              "      <td>19524</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65+</th>\n",
              "      <td>11541</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>100 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-35edfb28-3eb9-4d2e-8f80-0f162d5eaad4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-35edfb28-3eb9-4d2e-8f80-0f162d5eaad4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-35edfb28-3eb9-4d2e-8f80-0f162d5eaad4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define column mappings\n",
        "COLUMN_MAPPINGS = {\n",
        "    'region_name': 'state',\n",
        "    'variable_name': 'gender',\n",
        "    'age_name': 'age_category',\n",
        "    '2008': 'population'\n",
        "}\n",
        "\n",
        "# Define age group mappings\n",
        "AGE_GROUP_MAPPINGS = {\n",
        "    '0 to 4': '0-14', '5 to 9': '0-14',\n",
        "    '10 to 14': '0-14', '15 to 19': '15-29',\n",
        "    '20 to 24': '15-29', '25 to 29': '15-29',\n",
        "    '30 to 34': '30-49', '35 to 39': '30-49',\n",
        "    '40 to 44': '30-49', '45 to 49': '30-49',\n",
        "    '50 to 54': '50-64', '55 to 59': '50-64',\n",
        "    '60 to 64': '50-64', '65+': '65+'\n",
        "}\n",
        "\n",
        "def preprocess_census_data(df):\n",
        "    \"\"\"\n",
        "    Preprocess a DataFrame containing census data.\n",
        "    \n",
        "    The function cleans up the column names, handles missing values, renames columns,\n",
        "    converts columns to their appropriate data types, filters rows, remaps age categories,\n",
        "    and then groups the data.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    df : pd.DataFrame\n",
        "        The original DataFrame to be preprocessed.\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    pd.DataFrame\n",
        "        The preprocessed DataFrame.\n",
        "    \"\"\"\n",
        "\n",
        "    cols = ['Region Name', 'Variable Name', 'Age Name', '2008']\n",
        "\n",
        "    # Subset the data\n",
        "    preprocessed_data = (\n",
        "        df[cols]\n",
        "        .clean_names()\n",
        "        .dropna()\n",
        "        .rename(columns=COLUMN_MAPPINGS)\n",
        "        .assign(\n",
        "            population=lambda x: x['population'].astype('int'),\n",
        "            gender=lambda x: x['gender'].str.split(' ').str[1]\n",
        "        )\n",
        "        .query('gender != \"Total\" & age_category != \"Total\"')\n",
        "        .assign(\n",
        "            age_category=lambda x: x['age_category'].map(AGE_GROUP_MAPPINGS)\n",
        "        )\n",
        "        .groupby(['state', 'gender', 'age_category'])['population']\n",
        "        .sum(numeric_only=True)\n",
        "    )\n",
        "\n",
        "    return preprocessed_data\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CoYzA_UQT-Ja",
        "outputId": "6ef39176-0a69-43cd-a142-4868d1fd0c67"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "aa = preprocess_census_data(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NMkK38-yUDS0",
        "outputId": "ddad0f7f-200d-401b-9d0b-69c481af156c"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "aa.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rflSSMAUUF4a",
        "outputId": "9d5b6cf8-a4d5-416e-cae6-407678cef86d"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "state              gender  age_category\n",
              "Central Equatoria  Female  0-14            221216\n",
              "                           15-29           166887\n",
              "                           30-49           101676\n",
              "                           50-64            23460\n",
              "                           65+               8596\n",
              "Name: population, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "aa.tail()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zAtahy57UIFF",
        "outputId": "7774276b-0ca0-4696-f52b-3f4569fdb0c6"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "state              gender  age_category\n",
              "Western Equatoria  Male    0-14            125766\n",
              "                           15-29            93082\n",
              "                           30-49            68530\n",
              "                           50-64            19524\n",
              "                           65+              11541\n",
              "Name: population, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "m7ZkIK2FUKSu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}