2.1 - [Data Cleaning] Control variables.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\hideki\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2698: DtypeWarning: Columns (15,17) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21251\n",
      "2556\n"
     ]
    }
   ],
   "source": [
    "# Importing base_111 file\n",
    "base = pd.read_csv(\"base_111.csv\", encoding=\"latin-1\")\n",
    "# base2 = pd.read_csv(\"base_222.csv\", encoding=\"latin-1\")\n",
    "# base265 = pd.read_csv(\"base_265.csv\", encoding=\"latin-1\")\n",
    "dup_init = base.shape[0]\n",
    "base = base.drop_duplicates()\n",
    "dup_final = base.shape[0]\n",
    "print(dup_init - dup_final)\n",
    "\n",
    "\n",
    "cod_init = base.shape[0]\n",
    "base = base[base['COD'] == 111]\n",
    "cod_final = base.shape[0]\n",
    "print(cod_init - cod_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtUAAAF+CAYAAABNgObbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8ZFV16PHfopsxyNwi0AgOKDaO2AJKEsUBGlAgcQIH\nWoOQRJyiUcGXFxKMLyTREI1gREHACXEAiaLYgji8PJB2AoEYO4gBIqEFBCdUYL0/9i77eLndt6p2\nVd9b19/386nPrbPr3HXX3XVO1apz9tkVmYkkSZKk4W0w2wlIkiRJk86iWpIkSWpkUS1JkiQ1sqiW\nJEmSGllUS5IkSY0sqiVJkqRGFtWSJElSI4tqSZIkqZFFtSRJktRo4WwnMKztttsud91119lOQ5Ik\nSfPY1772tR9m5qKZ1pvYonrXXXdl5cqVs52GJEmS5rGI+H4/6zn8Q5IkSWpkUS1JkiQ1sqiWJEmS\nGllUS5IkSY0sqiVJkqRGFtWSJElSI4tqSZIkqZFFtSRJktTIolqSJElqZFEtSZIkNbKoliRJkhpZ\nVEuSJEmNLKolSZKkRgtnOwFJkgax63GfHlms6086eGSx5jv7XVo3i+rfIr4gzp5R9f1s9LvbzdpN\n8vM6Tm4zkn4bWVQPyDcLzTVuk/PTuJ9Xt5vZYb9L85dFtSRJHZ6BkDQMi2oJjx5J0nzma7zWB4tq\njYwvWpIk6beVRbUkSfOEBzek2eM81ZIkSVIjj1RLmrc8aidJWl8sqiXNGoteSdJ8YVEtSZJmnR+y\nNekcUy1JkiQ1sqiWJEmSGvVVVEfE9RFxVUR8MyJW1rZtImJFRHy3/ty6tkdEvCMiVkXElRGxZyfO\n8rr+dyNieaf98TX+qvq7Mep/VJIkSRqXQY5U75eZj83MpXX5OODizNwNuLguAxwI7FZvxwDvglKE\nAycAewN7ASf0CvG6ztGd31s29H8kSZIkrWctwz8OBc6q988CDuu0n53FZcBWEbEDcACwIjNvy8zb\ngRXAsvrYFpl5WWYmcHYnliRJkjTn9VtUJ/C5iPhaRBxT27bPzB/U+zcD29f7OwE3dH73xtq2rvYb\np2m/j4g4JiJWRsTK1atX95m6JEmSNF79Tqn3u5l5U0TcH1gREf/efTAzMyJy9On9psw8DTgNYOnS\npWP/e5IkSVI/+jpSnZk31Z+3AOdRxkT/Tx26Qf15S139JmDnzq8vrm3ral88TbskSZI0EWYsqiPi\ndyLifr37wP7At4ELgN4MHsuBT9b7FwBH1llA9gHuqMNELgL2j4it6wWK+wMX1cfujIh96qwfR3Zi\nSZIkSXNeP8M/tgfOq7PcLQQ+lJmfjYgrgHMj4ijg+8Dz6voXAgcBq4CfAS8FyMzbIuLNwBV1vRMz\n87Z6/+XAmcCmwGfqTZIkSZoIMxbVmXkd8Jhp2m8FnjZNewLHriXWGcAZ07SvBB7ZR76SJEnSnOM3\nKkqSJEmN+p39Q5pVux736ZHFuv6kg0cWS5IkCTxSLUmSJDWzqJYkSZIaWVRLkiRJjSyqJUmSpEYW\n1ZIkSVIji2pJkiSpkUW1JEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmSGllUS5IkSY0sqiVJkqRGFtWS\nJElSI4tqSZIkqZFFtSRJktTIolqSJElqZFEtSZIkNbKoliRJkhpZVEuSJEmNLKolSZKkRgtnOwH9\npl2P+/RI4lx/0sEjiSNJkqSZeaRakiRJamRRLUmSJDWyqJYkSZIaWVRLkiRJjSyqJUmSpEYW1ZIk\nSVIji2pJkiSpkUW1JEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmSGllUS5IkSY0sqiVJkqRGFtWSJElS\nI4tqSZIkqZFFtSRJktTIolqSJElqZFEtSZIkNbKoliRJkhpZVEuSJEmNLKolSZKkRhbVkiRJUiOL\nakmSJKmRRbUkSZLUyKJakiRJamRRLUmSJDWyqJYkSZIa9V1UR8SCiPhGRHyqLj8oIi6PiFUR8ZGI\n2Ki2b1yXV9XHd+3EOL62fyciDui0L6ttqyLiuNH9e5IkSdL4DXKk+tXAtZ3lvwNOzsyHArcDR9X2\no4Dba/vJdT0iYglwOLAHsAw4tRbqC4BTgAOBJcARdV1JkiRpIvRVVEfEYuBg4L11OYCnAh+rq5wF\nHFbvH1qXqY8/ra5/KHBOZv4iM78HrAL2qrdVmXldZv4SOKeuK0mSJE2Efo9U/xPwBuDeurwt8KPM\nvLsu3wjsVO/vBNwAUB+/o67/6/Ypv7O29vuIiGMiYmVErFy9enWfqUuSJEnjNWNRHRHPBG7JzK+t\nh3zWKTNPy8ylmbl00aJFs52OJEmSBMDCPtbZFzgkIg4CNgG2AN4ObBURC+vR6MXATXX9m4CdgRsj\nYiGwJXBrp72n+ztra5ckSZLmvBmPVGfm8Zm5ODN3pVxoeElmvhD4AvCcutpy4JP1/gV1mfr4JZmZ\ntf3wOjvIg4DdgK8CVwC71dlENqp/44KR/HeSJEnSetDPkeq1eSNwTkT8DfAN4PTafjrw/ohYBdxG\nKZLJzKsj4lzgGuBu4NjMvAcgIl4BXAQsAM7IzKsb8pIkSZLWq4GK6sy8FLi03r+OMnPH1HXuAp67\nlt9/C/CWadovBC4cJBdJkiRprvAbFSVJkqRGFtWSJElSI4tqSZIkqZFFtSRJktTIolqSJElqZFEt\nSZIkNbKoliRJkhpZVEuSJEmNLKolSZKkRhbVkiRJUiOLakmSJKmRRbUkSZLUyKJakiRJamRRLUmS\nJDWyqJYkSZIaWVRLkiRJjSyqJUmSpEYW1ZIkSVIji2pJkiSpkUW1JEmS1MiiWpIkSWpkUS1JkiQ1\nsqiWJEmSGllUS5IkSY0sqiVJkqRGFtWSJElSI4tqSZIkqZFFtSRJktTIolqSJElqZFEtSZIkNbKo\nliRJkhpZVEuSJEmNLKolSZKkRhbVkiRJUiOLakmSJKmRRbUkSZLUyKJakiRJamRRLUmSJDWyqJYk\nSZIaWVRLkiRJjSyqJUmSpEYW1ZIkSVIji2pJkiSpkUW1JEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmS\nGllUS5IkSY0sqiVJkqRGFtWSJElSoxmL6ojYJCK+GhHfioirI+Kva/uDIuLyiFgVER+JiI1q+8Z1\neVV9fNdOrONr+3ci4oBO+7Latioijhv9vylJkiSNTz9Hqn8BPDUzHwM8FlgWEfsAfwecnJkPBW4H\njqrrHwXcXttPrusREUuAw4E9gGXAqRGxICIWAKcABwJLgCPqupIkSdJEmLGozuIndXHDekvgqcDH\navtZwGH1/qF1mfr40yIiavs5mfmLzPwesArYq95WZeZ1mflL4Jy6riRJkjQR+hpTXY8ofxO4BVgB\n/Cfwo8y8u65yI7BTvb8TcANAffwOYNtu+5TfWVv7dHkcExErI2Ll6tWr+0ldkiRJGru+iurMvCcz\nHwssphxZ3n2sWa09j9Myc2lmLl20aNFspCBJkiTdx0Czf2Tmj4AvAE8EtoqIhfWhxcBN9f5NwM4A\n9fEtgVu77VN+Z23tkiRJ0kToZ/aPRRGxVb2/KfAM4FpKcf2cutpy4JP1/gV1mfr4JZmZtf3wOjvI\ng4DdgK8CVwC71dlENqJczHjBKP45SZIkaX1YOPMq7ACcVWfp2AA4NzM/FRHXAOdExN8A3wBOr+uf\nDrw/IlYBt1GKZDLz6og4F7gGuBs4NjPvAYiIVwAXAQuAMzLz6pH9h5IkSdKYzVhUZ+aVwOOmab+O\nMr56avtdwHPXEustwFumab8QuLCPfCVJkqQ5x29UlCRJkhpZVEuSJEmNLKolSZKkRhbVkiRJUiOL\nakmSJKmRRbUkSZLUyKJakiRJamRRLUmSJDWyqJYkSZIaWVRLkiRJjSyqJUmSpEYW1ZIkSVIji2pJ\nkiSpkUW1JEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmSGllUS5IkSY0sqiVJkqRGFtWSJElSI4tqSZIk\nqZFFtSRJktTIolqSJElqZFEtSZIkNbKoliRJkhpZVEuSJEmNLKolSZKkRhbVkiRJUiOLakmSJKmR\nRbUkSZLUyKJakiRJamRRLUmSJDWyqJYkSZIaWVRLkiRJjSyqJUmSpEYW1ZIkSVIji2pJkiSpkUW1\nJEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmSGllUS5IkSY0sqiVJkqRGFtWSJElSI4tqSZIkqZFFtSRJ\nktTIolqSJElqZFEtSZIkNbKoliRJkhpZVEuSJEmNZiyqI2LniPhCRFwTEVdHxKtr+zYRsSIivlt/\nbl3bIyLeERGrIuLKiNizE2t5Xf+7EbG80/74iLiq/s47IiLG8c9KkiRJ49DPkeq7gddl5hJgH+DY\niFgCHAdcnJm7ARfXZYADgd3q7RjgXVCKcOAEYG9gL+CEXiFe1zm683vL2v81SZIkaf2YsajOzB9k\n5tfr/R8D1wI7AYcCZ9XVzgIOq/cPBc7O4jJgq4jYATgAWJGZt2Xm7cAKYFl9bIvMvCwzEzi7E0uS\nJEma8wYaUx0RuwKPAy4Hts/MH9SHbga2r/d3Am7o/NqNtW1d7TdO0z7d3z8mIlZGxMrVq1cPkrok\nSZI0Nn0X1RGxOfBx4DWZeWf3sXqEOUec231k5mmZuTQzly5atGjcf06SJEnqS19FdURsSCmoP5iZ\nn6jN/1OHblB/3lLbbwJ27vz64tq2rvbF07RLkiRJE6Gf2T8COB24NjP/sfPQBUBvBo/lwCc77UfW\nWUD2Ae6ow0QuAvaPiK3rBYr7AxfVx+6MiH3q3zqyE0uSJEma8xb2sc6+wIuBqyLim7XtTcBJwLkR\ncRTwfeB59bELgYOAVcDPgJcCZOZtEfFm4Iq63omZeVu9/3LgTGBT4DP1JkmSJE2EGYvqzPwKsLZ5\no582zfoJHLuWWGcAZ0zTvhJ45Ey5SJIkSXOR36goSZIkNbKoliRJkhpZVEuSJEmNLKolSZKkRhbV\nkiRJUiOLakmSJKmRRbUkSZLUyKJakiRJamRRLUmSJDWyqJYkSZIaWVRLkiRJjSyqJUmSpEYW1ZIk\nSVIji2pJkiSpkUW1JEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmSGllUS5IkSY0sqiVJkqRGFtWSJElS\nI4tqSZIkqZFFtSRJktTIolqSJElqZFEtSZIkNbKoliRJkhpZVEuSJEmNLKolSZKkRhbVkiRJUiOL\nakmSJKmRRbUkSZLUyKJakiRJamRRLUmSJDWyqJYkSZIaWVRLkiRJjSyqJUmSpEYW1ZIkSVIji2pJ\nkiSpkUW1JEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmSGllUS5IkSY0sqiVJkqRGFtWSJElSI4tqSZIk\nqZFFtSRJktTIolqSJElqZFEtSZIkNbKoliRJkhrNWFRHxBkRcUtEfLvTtk1ErIiI79afW9f2iIh3\nRMSqiLgyIvbs/M7yuv53I2J5p/3xEXFV/Z13RESM+p+UJEmSxqmfI9VnAsumtB0HXJyZuwEX12WA\nA4Hd6u0Y4F1QinDgBGBvYC/ghF4hXtc5uvN7U/+WJEmSNKfNWFRn5peA26Y0HwqcVe+fBRzWaT87\ni8uArSJiB+AAYEVm3paZtwMrgGX1sS0y87LMTODsTixJkiRpIgw7pnr7zPxBvX8zsH29vxNwQ2e9\nG2vbutpvnKZ9WhFxTESsjIiVq1evHjJ1SZIkabSaL1SsR5hzBLn087dOy8ylmbl00aJF6+NPSpIk\nSTMatqj+nzp0g/rzltp+E7BzZ73FtW1d7YunaZckSZImxrBF9QVAbwaP5cAnO+1H1llA9gHuqMNE\nLgL2j4it6wWK+wMX1cfujIh96qwfR3ZiSZIkSRNh4UwrRMSHgacA20XEjZRZPE4Czo2Io4DvA8+r\nq18IHASsAn4GvBQgM2+LiDcDV9T1TszM3sWPL6fMMLIp8Jl6kyRJkibGjEV1Zh6xloeeNs26CRy7\nljhnAGdM074SeORMeUiSJElzld+oKEmSJDWyqJYkSZIaWVRLkiRJjSyqJUmSpEYW1ZIkSVIji2pJ\nkiSpkUW1JEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmSGllUS5IkSY0sqiVJkqRGFtWSJElSI4tqSZIk\nqZFFtSRJktTIolqSJElqZFEtSZIkNbKoliRJkhpZVEuSJEmNLKolSZKkRhbVkiRJUiOLakmSJKmR\nRbUkSZLUyKJakiRJamRRLUmSJDWyqJYkSZIaWVRLkiRJjSyqJUmSpEYW1ZIkSVIji2pJkiSpkUW1\nJEmS1MiiWpIkSWpkUS1JkiQ1sqiWJEmSGllUS5IkSY0sqiVJkqRGFtWSJElSI4tqSZIkqZFFtSRJ\nktTIolqSJElqZFEtSZIkNbKoliRJkhpZVEuSJEmNLKolSZKkRhbVkiRJUiOLakmSJKmRRbUkSZLU\nyKJakiRJamRRLUmSJDWyqJYkSZIazZmiOiKWRcR3ImJVRBw32/lIkiRJ/ZoTRXVELABOAQ4ElgBH\nRMSS2c1KkiRJ6s/C2U6g2gtYlZnXAUTEOcChwDWzmpUkSdIMdj3u0yOLdf1JB48sltavyMzZzoGI\neA6wLDNfVpdfDOydma+Yst4xwDF18eHAd9Zrov3bDvih8dd77HHHn+Tcxx3f3OdnfHOfn/HNfX7G\nN/fx2SUzF8200lw5Ut2XzDwNOG2285hJRKzMzKXGX7+xxx1/knMfd3xzn5/xzX1+xjf3+Rnf3Gff\nnBhTDdwE7NxZXlzbJEmSpDlvrhTVVwC7RcSDImIj4HDgglnOSZIkSerLnBj+kZl3R8QrgIuABcAZ\nmXn1LKfVYtxDVCY5vrnPz/jmPj/jm/v8jG/u8zO+uc+yOXGhoiRJkjTJ5srwD0mSJGliWVRLkiRJ\njSyqJUmSpEYW1bOkfjX7OOOP9bldD/nHGGOPu2/GHX+cfTO22DX+2PrG53WdsSeqbyJig17McfTL\nuONP/VuTFHucfePzuu54E5z7xhGx8ShjTom/3vqmlRcqrkcRsQuwYWauGlP8HYBfZuatY4o/tvwj\n4hnAfsB3gZWZeVVEbJCZ944g9sHAwcDNwEWZeXlrzPUcf5x9M7bYNf7Y+sbndZ2xJ7ZvIuIw4DnA\nPcD7M/Pzo4q9nuJP8jY/tr7xeV1n7EnO/dnAi4HNgVOBL46yBhl334yaR6rXk4j4A+BS4OSIWBER\nT4qI3xlh/EOBLwLvjIhPR8T2o4pd448t/4h4MnAKcDvwQOD8iHhqZt7b+ok6IvYB/hH4KnAn8KmI\neFZrzusx/jj7Zmyxa/yx9Y3P6zpjT2zfRMSjgb8FPgB8AfiniHhJRNxvQuJP8jY/tr7xeV1n7EnO\n/aHAicBJlNez5cBLIuJhI4o/1r4Zi8z0NuYbsBXwKWCvuvwXwOnAHwK/M4L4OwBfBvapy+8GPgY8\nfkLyfxnw953l51OKjf3qcjTEfjbwvs7yIcA1wDNbY6+n+OPsm7HFHnff+LzOz74Bngr8a2d5P+BC\n4MgR5T7u+JO8zY+tb3xe523ujwe+1FleSqk/Xgts2ZL3+uibcdxmPYHfhhuwIfAZ4AWdtj+mFKZP\nqMsLGuJvAvwr8Hudtr8GPg5sV5c3mMP5Pwt4z5S25wH/Djyise/3BN4L7NBpOwS4BVg6gud23PHH\n2Tdjiz3uvvF5nZ99Azygvq48sfeaUt9Yr6V+4Jjj8Sd5mx9b3/i8ztvcA3gPcASwUW1bSqkXDpnr\nz+s4bg7/GLOIiMz8FXA28LCIeARAZr4buJFyaoPMvGfY+MCvgH8DHhUR29Z4JwA/As6oy0ON0xx3\n/tUlNfeTew2ZeS7lQ8GShrgAV1PGer2+XuwQmXkB8A/Avo2x10f8cfbNOGPDePvG53XtJrlvfgis\nBp4L7BIRCzPzEsrp68MaY6+P+JO8zY+zb3xe126Scwf4GvAE4PciYuPMXAm8D1geEa3f2j3uvhk5\ni+oxqhtv1sWrgPsDyyJiCfy68P1pROw47N/I4h7K8I8Da/zt6mNHAXe0jH0ed/4RsSAzfwosA57Z\nLTIoR+Af0RB7g8z8BXAM8DjgrcAundgPHDb2OOPXD0pj6ZtO7A3G0e+x5grtsfX9OGJH58r1SXxe\n13Pu43peIzPvBt4MbA28Avjd+vCGwKaNuW8w5vgj75tx70+d+CPt++hcFzCO5zUiNh9n/F7c+nMc\nz+tYt8nO3xnpdhMRG3WXa31wJuUA3kGUI9YAGwO/BJIBra++GRdn/xiDOkj/5sy8c0r73pSrZO8A\nVgG/AP6KMhb6tiH/1q9nC4iIZcCfUo6CXQ1sDxwP7JuZdwwZ/9cfDEaRf0RsBtxTd/Re24aZ+auI\n2IoyjOU7lA98+wDPzsxrB8y52ycLMvOeKBc2nErZ0bcCdgeen5nfHiDuJsC9mfnLTtso428B/Coz\nf95pW5iZd7f2TUQ8GPhhZt7Ze047uTf3e0TsBVyXmT/stI2kbyLiqcDPgMs72+Io+30/YI/MfGcn\n7ijjPxD4MfCTetaHUfV9RGxK2Z+622RvmxlF7tsCd9UPAb223v7aHL/GW9A709Xbd+uBgDcCi+tt\nR+CFmfmtAeJuRZkN6WdT/9aI4j8A+CnleR3pdhkRe1DeQ27t9Mkot8nfB/4jM2/utI2k7yPi6ZQj\no5/NzLtGGbvG2o/ygfR/UV6P7x1x/IcDP6FsO6tr26ie120o+1N3m+ztr6PIfWyvB1FmK3oI8OFe\nPdHp902AFwBPAh4GbAm8JDO/MUDuY+2b9SbnwBiU+XQD9qdcBHBwXV4w5fHdgT8APgGcCzx2wPhP\nBv6ccmHTlrUtOo8/Efgz4DxgBfCYAeM/lnLEezGw2dT/oSV/yoWNn6AU/c8EHtx5rDcea7P6P74I\neNgAsX8f+KPO8gad+wvrz40pO/zBwIMG7JdnAh+quT+l+zdGGP+C+pw9Y8pjGzb2zZ7ADyhDdbbp\nbjOtsTvb/FXTbWud+EP1DeXN80bggE5bL/eNWmJ3cr+N8gFx8ZTHRvG8HkIZmvU5ypvMdPkP+7we\nWrfJj9f/Y5fOY6Pomz+seV8KHA3sPar4Nd/jO8vT7a8LKW/O+wI7DZj7sygXV3+R8mZ/vxHHP5jy\nWvBp4I+ABdPsU8P2zRLKeNcPA9t3+6c1dqfvv9d9PjuP9eIP1TfAAZT99Rmdtqnbe0u/H0B5Lfsx\nsPuU+KPablZS3j//ou6bG0zJf9jn9VDKB+hPUA5OLe08Noq+GdvrAfAM4Cbg6dM81ovdex4eCSwa\nIvex9c36vM16AvPpVjfkb1CmfnnflMdiyvICYNMB4z8L+CZlLNQHuht478Wws7yQAa++pRQAV1Eu\nPHgrcDJrLnRc2JI/8CDgPyhjr15EmX7nBOBx6+qnAfr9dsob6Os67RtMWW+LIZ/XZbVffg84ljKG\n7OHTrDds/GcCX6ccpTwa+H9MM6vKMH1Tf28r4CJKUX3SdC94DbEPqbnv2dsu1rLeQH1DuQBmc8qH\njANr2+/UN4VNRtzvSyhndE7qvYCPKP5jgW8Dj6p/63N0iruWvgceU7fJR1M+5H6o7rNLRpT7jpSj\n53vW/etNwL9w3w98A8enfAC+hXJR5ls77VP316HeOCkFwLcpF0w9lzJbwHQF5LDxD6a8zj+Bcsr7\nEmDrEW43C4FzgLdTDlwsHmHsA4BvsWamqI2n9vswfVP3100oBd2za9uW9baoJXbn955V99eHAK+k\nzHC1+Qif14d39tc9KYX1ttNsl8Ns8w+r++uSuv3/A+VD0++OKPexvB7U53Uh5ULHF9e2berrw0Om\n9t+QuY+1b9b3zTHVI1JPUb+bcjRqP+CREfGi3uPZ20IjnhgRSzLznuyc5u8j/mbAkcDRmfl64PvA\nHhGxe0Rsk2tOKz86InbJzLtzgCEf9bTRC4DlmXk05SjPvsDbIuL+WcY2DZ0/sAVwY2ZekZkfoFzI\nsBB4Vj09TkQ8jTLH5aDf1vhQ4O+B1wBPjIjXQbk4szNmcG/KhRObDBK4rn8A8ObM/HJmngJ8lnI0\nv7ve3sCRQ8TfEHgK8KbMvIxy1fQvgb+KiGdEHa8+bN/U/39hjfk9yovky6LMM/64ltjVcygvdl+v\nw1f+PiLOjog/6OQ+cN/U/eUuylGpi+qp/HOADwIn1lPMLf2+NeVI7Osz8xpKgbcL5cNid1zfUPGr\nBwLXZOZVwJco+8A7IuLlEfHYGn/Yvt+lxr4yM8+jHE3em7I/LRpB7guA/8rMr2fm5yh9/y3gDyPi\n8Y3xd6Scut8XeFxEvA3us7/uAbw4BpyPtvbjvpRifWVmfpTS98+tj/ee16HiV48H/ndmXkEp8rYE\n/i4iXhhlXt2W7XIDSnEK5Tm9mrK97x9lbvPW5/XplIMhl9Xt5J3AhyPi2E7uewAvHKRvsriL8r50\nWZQxz+cDp1HmFj6iE/tFQzyvm9Tc35CZ/wl8hTJEY/v6eHe7GSj3ju0o71FXUQ4APYRy8OdvWl9v\nOrGvycwv1fy3AZ4f9RqlxtzH8npQn9e7Ke8d19VhGBcCfwe8NyJeVWPvUnPfeojc7894+2a9sqge\nnZspU859q75wfwDYDX7jhXwLYA/KoP5BJeVT85OjfHPi8ynF2HHAuyJi+1p47E8ZfzpM/AdQPqWT\nmZdSjor/BPiTiFhQ4w+Vf5axTz+KiFfW5ZWU4Q47s+bCrBuAS3LAmUQy81TKWLFvU94k9omI19fH\nem/U3wc+Vl/4B4l9V4352U7hcyf1ue24Afj4EPF/BRyXmZ+tLxgfBy6jFDAvoryR9OIP0zf3Zhnn\n/BnK5P+nsOYozHZ1tRuHiV3jHwl8JSL+nXL67r+AKyhzoz6lrvYDhuubuynfonUa8E/ARylHk79P\nebPYlLLfDRybsg2/IjMvrn/rPMr2/w91Oet6Nw0ZH0p/bxcR51KmgLqAcnpzZ9Z8KBu2768CfhUR\nR9blB9S/8RjgwbXtv4fNPTNvAG6LiLfW5esoR9r/h3J6F4Z/Xs+h7Iu3AkcBj4l6sWauGRf7X5Qp\nB388YOx7KK8F50VF6Yf718ez7sc3Au8dNH6NcWJmfirKgY7zKENAzqf0yzPrasP2zb2Z+RPK/vqL\nzPxrSpEhrbcdAAAOfklEQVT9Udbsr8Nu89QDMl+MiCuAT1KOuH+C8gFwWV1tNXDGIH3Te4+r3ku5\nuOx9lPmKP0r5MLYj5QzF6UM8r3dRCuret+l9k3Lm6sT6eG92q5sHzb3jCmDTiPg88J+U9/B/pGw/\nh9Xne9i+/zZwZ0T8RV1+HOVM0F2suVjwlobcrwTujogX1+WRvh5QDsacRBla+m7gJcAbKIXvHsCt\nlA+ytw8R+2vAz8bYN+vXqA55/7beKFehbtxZ7o0r2hO4nilzKTLN6eU+4vfGFD2B8ka9AnhLbdsV\neBdrxnAP9GUsU+IfTBm68ibKqaOPUYq6DwyTP6Woeh5rThsdVuMe3lnnjyjFxsYD5t2L/aIp7ZtQ\nJoj/KPBSyhGq5UM8r0+hfHB53jSPHUz94o66zrKG+C+Y0r575/7hlDMGw/ZNd17x4ylDS36fUqCf\nB7yFAce+TYm/vNP2YeBvO8tHUIrsQbf3qdvMLpRhSFf0+oFyuvB8YNuGfj+i09YbM7mEctr0Ed32\nIfM/opP/IZQisrfO71OKpvucuu4z9uF1+WV1H/0M8Ona9nLKsIFhhpQspjNkjDJ85VTgz6ds+xdR\nr7cYNvaUxx4MfB74y7q/Hs+A895340993ihH7E7vbJfLB+2fKfG717As7tzfr+6vgw7ru0/udV99\nI+Wo+/eAj9TneuBT4NP1PfDPwP/qLD+15r5JS2zKGY5TKEd6d6xtW1JeHx44ZO5bTWnr9dG2lKOm\n+099Xhry34Qyw8T7O227U96jtmt4XhdShiWdX2N9srM9njV1m+0z/lPqtnxUXX4p5XX9wtbXgxr7\naOBVnbb3MWVua8oHqD2GyP3pwOsoH5R6f++CUfXNbN48Ut0gyleDnwl8sJ6qv1/WrSEzvw68jXJq\nd4ve72TnqtwB4n8oIg7KcrrxSZThB9+r8a4HNqLswGTnKv0B4z8DuJxyWnYb4I7MfE6WIwNbRx2i\n0W/+Ua7Q/jDlk+ZrIuIfKZ8+vwc8IeoQDeDnlItOctpA/cU+NSJ2qvndRbkw7C8pb0rvpXwS7lsn\n/s7AGyPiXfGb0wb+ErgnIl5AOVLyvYb4r+3Fj4jIzH+fctRnoLMCU/rmtRHxL/WU3PmUMeEfBF5F\nGc9+L+UIxLDxXxkR74mIrTLziMw8vpP7BpRZYoaN/ZqIeGeN85ma6z/XVZdQxokP9Po1pd9fV7eb\nHXPNUa5bKW9+vwuDz+0+Jf/XR8QpwM+zzAl7c91eoBQZC+rfGib2G+oR5M9TPpS+ijVzti6k7Lt9\n7081/mE13lG908WUITH/CjwkIt5e2zanbP99bzdTYm9X2379+1mOgh9E+UKp04BP5QBH7qfGn+Z5\nuwe4NyJeQnlduHyQ/pkmfkad75dyJqNnW8p2Osjzurbcz6ccsfsE5WjvSyhnrwbdX+/T9wCZ+UrK\nkcee7Sj9NHTutfle4B3UI9K1jw6gHPy5e8jc/2hK7vfWuD+jHLF+bG1v3uYz867M/Arwk4g4rq76\nUMrrzbDb/KIswzFXUIacHVN/AtwP+NEQrzUHUc6gbgKcEBEvo3yXxAspExkM/XpQY59KOeD2ZxHx\nnvrQyyjDnf4lIu5f96c9Ke/fg+b+T5Sj/q+LiOMy89LMPAT4E8pZThiyb2bdbFf1k3qjHDX+JmWj\neiblIrm/ovNpnPKieAmdq3Ab4n+JUsBtRTmls4LySfL5lKLxoSPI/0SmXHBDuRL3G1PbZ4gdlDHO\nf1aXN6EeyaRcDHIg5U3jYspMKY9rjP1Bygv5Azrr/THlFPKSfmP3G5/yJvFTSvE+0LffzRB/+856\nL6Nchf7IEcQ+mTJs5+2suegvGPysxtriv3NK7kfX3B/VGLt3sc1iytHp91OO2K0EHj3q57W2P58y\njnVTBji6M0Pf7FSfz09QPiBcOUj+69if3kHn6BnlzfRbDHjkCFhU98X3UU7bv5J6BqP+rT1qv19I\nGXYyyP46NfYrmOaIH2Vs/vUjyP3X8Wu/BeX18seU/XX3UcWfst7LKa/Dg2zz68p9c8pBgSd3/pdB\nz/r0m/uxDL6/Thd7UefxTSgfkN4N/N9BYg+Y+zLgOspQkEH213XGpwyjvIBy5nbQ/XWt+9OU9V5T\n99dB+2a3+nztV5dfSJlKd+mU9QZ+PaB8aP834Gl1eUvKGOdHdNZ5K+Us5+eG2F93pgxvfHrn+ftL\n6tmG1r6ZC7dZT2BSb5RPU2d2lv+EUnwezW9OQfd2BpzyaB3xv0n5lLsZ5ZPo5yhvdgMVGGuJ/6c1\n/2MoRwA3qOtcPmT8wymzBfSmhNqMciX7yZ11dmO4U/jTxT4HOLWzzmuG3SFnik+5OOaSQV9QBoj/\ncMoR9oHzX0e/d4dmtHyl/Ey571gfH1XuHwH+ubPOtgw/88GM201t7/sDZJ/5/0NdfhBlBoNhToP3\ns82fyIBTaNbf24hyLcXGdZ9/O6UQeMCU9bZn8BmF1hb7/vXx3qn85Qzx9ewzxe/01aeH3CZnyn9h\n3ebfPmj8mfqdzlRiQ26PM/Y95eLZt40w9+2nrLcJA354H2S7qfcH2ib7yb8+vgXlrNUOI869N0T0\ndQxw0KQTf3Hv9+o++UPKMIlLaXw9qPGWdf6PDSkH8J44zboDDRWqv3M/6lS6lHHft1I+fK2gHIDY\ntD72WoZ8f53t26wnMKk3ynRNZ7BmPNebKN9R/xXKl62MPX7daQcabztg/EXAzgPE3LnmtCllLOkH\nKePIejvKZpSjOX8wRL4zxd60xj50yP7oN/6z6gvNoMVFv/EPorzZDTJdYT/9/vX10DcHU46o9b1N\njnObGTD3Q8YUfzPKh9WD5+A2/0DKG+dmU9qfTTkK/sq6vHSMsfs+6t0Sf5BtcsD4j64/+y5851Df\nPKb+7HvM6gCxHz/m3Af6fof1EX+A2Hs25L5hZ//fkHJE/Xl1eVPKl7ItHyJ29zqBqWPY302dkpLy\nPRgDH5SZEr/3QfpR1Nd1yuvc5cCrh+mbuXRzTPUAImKviNg3Ih6TZfaKa4GjI+JCSiF6FGVc0/5j\njP/+XvzM/EV2vplw1Pln5uosswD0E/tgyqntf6YU67+knKJ+NfB7EbFDlm9Kurg+1rc+Y/+8xh5o\n3N4Q8cnMX+Vg0xUOEn+DLDMA9DVd4QD9/nnG3zeRRV/b5Di3mSFyH3jmkwHyXzHm3Id9Xi+kHB16\nX0Ts3nssMz9OGQ62KCLOB74w5ZqCUcb+Uu96iDHF/3Ltq0FeJweJ/291XH5fz8EQfdN3vw8R/ytT\nrikYZexLx5z7l+dS/AFjf7Fhmz8FODMids8yc9SXM/PcKN/4+HPKkLlBr8M5mDJ2+n413x/V9l59\nuBVlVpQjKPXHomkD9R+/t719OzPPizVfpf4Jymxjk222q/pJuVGOwn0L+D+UI0Zvre33o1wo0fsm\nqjdS5jD9rYhPOTK5M2Ws5VMop4/eQJlhYqf6d8+ut7+lTGXV17fGjTP2pMc39/kZf5Zy/3PKFHB7\nTFn3A5Rxzn0NDRhn7EmPb+7zM/4s594bAtI78nsE5axY39dXUa6t+inlCPcxdIbqdOK+lXJg4ItT\n/6ch4k874xFrrt0a6gtk5tJt1hOYhBvlFO5nWDN4/4GUsUCnT1nvVZT5KAe9eG3S4y+gjIvaiTXj\nxV5LmU94B8qYw0MpFyQMtNOMM/akxzf3+Rl/lnJ/NWUmi4fV5R0oFxEPdBp8nLEnPb65z8/4cyT3\nF1CGgg00RpsyDeT+lAkLvkC5aHXzKev0vmxuoIt8+4lPOah3CANelD+Xb7OewCTcKFcWn9t90ilf\nEnE18La6vCll9o9hLhSayPiUqYaeQLl47CPUOSc7jx9POXU98Ljvccae9PjmPj/jz4Hc30CZYrM3\nZrPvebTHGXvS45v7/Iw/h3LfgPItn/f5OvsZYj+aMrvHtrVtb8oF+K/o5UoZ6/xgBry4eoD4GwJb\nM+Wi6Em+zXoCc/lG57QqZV7fGylfvvBuytipBwPvoQ7sZ/AJ1ic2PmUavispp4TeSfm0eT1wfGed\nXSmfsAfNe2yxJz2+uc/P+HMo93ePMfeBY096fHOfn/HnUO6nNcS+lDIU9FGdx/ahFL6HU2YsGfjL\nVwaMfzYNs1HNxdusJzBXb3XD+BnwkU7bq4G/oHzvfW8M8icZcMqdSY9P+QKaa1lzZf1pwN9QTkv/\nV/0bD6V8YcFKBpvjemyxJz2+uc/P+OY+P+Ob+/yMP89yP5Xy9d+wZmjJzpQx2zcw+PcBjDX+JNxm\nPYG5eKMMl/gsZWD9mcCH17LeiyhT0A369aWTHv9JwEs6y4tY87WoD6acoj6VAb8MYdyxJz2+uc/P\n+OY+P+Ob+/yMPw9zP58yzGODzjo/YrivHx9r/Em4zXoCc/VG+VS4OeXrWz8GfKjz2ELKNwF9leHn\ny5zY+JQLJ7bo3F9MuXJ3h9q2S/0bw0zKP7bYkx7f3OdnfHOfn/HNfX7Gn6e5975BdUfKxYUPGXHu\nI4k/CTfnqV6LzPzvzPxJZv6Q8pXXG0fEB+rDu1MmeT8kM7/52xY/M+/JzDvrYlA+dd6WmT+IiBdR\nvkhmwxxgLuf1EXvS45v7/Ixv7vMzvrnPz/jzNPfVEfFC4Djgq5n5nyPOfSTxJ0FvjItmEBHbUWbM\neBLlatsnZ+Z/G//X8c+kjJPan3L656pJiD3p8c19fsY39/kZ39znZ/x5lPtLM/PKUcVeH/HnGovq\nAUTEn1G+HOUZo95pJjV+RARlWpxr68+nZeZ353rsSY9v7vMzvrnPz/jmPj/jm/vsxZ+rLKr7FBFb\nU+Z6ft04PmnNg/gvAa7IzKsnKfakxzf3+Rnf3OdnfHOfn/HNffbizzUW1QOIiE0y8y7jTxs7ckwb\n0zhjT3p8c5+f8c19fsY39/kZ39xnL/5cY1EtSZIkNXL2D0mSJKmRRbUkSZLUyKJakiRJamRRLUmS\nJDWyqJYkSZIaWVRLkiRJjf4/VVAmF9GGbPwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x18e3015b5c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "volumetry = base.drop_duplicates().groupby('YEAR').agg({'DAY365': 'count'}).reset_index()\n",
    "\n",
    "fig = plt.figure(figsize=(12, 6))\n",
    "# plt.title(\"Volumetria base original\")\n",
    "N = np.arange(len(volumetry['DAY365']))\n",
    "plt.bar(N,volumetry['DAY365'].values)\n",
    "plt.xticks(N,volumetry['YEAR'],rotation = 45)\n",
    "# plt.savefig(\"fig/CEPAGRI_volumetria\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The year 1998.0 has 49 missing days \n",
      "The year 2000.0 has 2 missing days \n",
      "The year 2001.0 has 24 missing days \n",
      "The year 2002.0 has 1 missing days \n",
      "The year 2006.0 has 9 missing days \n",
      "The year 2007.0 has 7 missing days \n",
      "The year 2019.0 has 364 missing days \n",
      "The year 2039.0 has 364 missing days \n",
      "The year 2126.0 has 364 missing days \n",
      "The year 2139.0 has 364 missing days \n",
      "The year 2013.0 has 1 missing days \n",
      "The year 2015.0 has 4 missing days \n",
      "The year 2017.0 has 7 missing days \n",
      "The year 2018.0 has 2 missing days \n"
     ]
    }
   ],
   "source": [
    "missing_years = []\n",
    "for v in base['YEAR'].unique():\n",
    "    if(base[base['YEAR'] == v]['DAY365'].nunique() < 365):\n",
    "        print(f\"The year {v} has {365 - base[base['YEAR'] == v]['DAY365'].nunique()} missing days \")\n",
    "        if(v < 2019):\n",
    "            missing_years.append(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So years 1998. 2000, 2001, 2002, 2006, 2007, 2013, 2015, 2017 and 2018 doesn't have all days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "52560"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "24*6*365"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "24*6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Year: 1998.0 missing days: [47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95]\n",
      "Year: 2000.0 missing days: [361, 362, 365]\n",
      "Year: 2001.0 missing days: [1, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281]\n",
      "Year: 2002.0 missing days: [253]\n",
      "Year: 2006.0 missing days: [144, 145, 146, 147, 148, 149, 150, 151, 152]\n",
      "Year: 2007.0 missing days: [319, 320, 321, 322, 323, 324, 325]\n",
      "Year: 2013.0 missing days: [209]\n",
      "Year: 2015.0 missing days: [17, 18, 172, 173]\n",
      "Year: 2017.0 missing days: [70, 71, 119, 120, 121, 122, 124]\n",
      "Year: 2018.0 missing days: [300, 301]\n"
     ]
    }
   ],
   "source": [
    "missing_days = []\n",
    "for y in missing_years:\n",
    "    missing_days.append([i for i in np.arange(1,366) if i not in base[base['YEAR'] == y].DAY365.unique()])\n",
    "    print(f\"Year: {y} missing days: {[i for i in np.arange(1,366) if i not in base[base['YEAR'] == y].DAY365.unique()]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Describe data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1138228, 18)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COD</th>\n",
       "      <th>YEAR</th>\n",
       "      <th>DAY365</th>\n",
       "      <th>MINUTE2400</th>\n",
       "      <th>N_WNDVEL_5M</th>\n",
       "      <th>N_WNDVEL_2M</th>\n",
       "      <th>N_WNDDIR_DEGREE</th>\n",
       "      <th>N_INCRAD_10MSR</th>\n",
       "      <th>N_REFRAD_10MSR</th>\n",
       "      <th>N_PHTFLOW_10MSR</th>\n",
       "      <th>N_NHTFLOW_10MSR</th>\n",
       "      <th>N_AIRMTR_PCT10</th>\n",
       "      <th>N_AIRTMP_MED10</th>\n",
       "      <th>N_GRDTMP_3CM</th>\n",
       "      <th>N_GRDTMP_6CM</th>\n",
       "      <th>N_PRESSION_HPA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1138228.0</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138228e+06</td>\n",
       "      <td>1.138227e+06</td>\n",
       "      <td>1.138227e+06</td>\n",
       "      <td>1.138227e+06</td>\n",
       "      <td>1.138227e+06</td>\n",
       "      <td>1.138227e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>111.0</td>\n",
       "      <td>2.007567e+03</td>\n",
       "      <td>1.834570e+02</td>\n",
       "      <td>1.191838e+03</td>\n",
       "      <td>1.338263e+03</td>\n",
       "      <td>8.314822e+02</td>\n",
       "      <td>2.618663e+02</td>\n",
       "      <td>7.067442e+02</td>\n",
       "      <td>2.609247e+02</td>\n",
       "      <td>3.430510e+02</td>\n",
       "      <td>-2.136187e+03</td>\n",
       "      <td>7.328247e+01</td>\n",
       "      <td>2.550711e+01</td>\n",
       "      <td>1.834196e+03</td>\n",
       "      <td>1.804436e+03</td>\n",
       "      <td>9.460472e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "      <td>6.320072e+00</td>\n",
       "      <td>1.054385e+02</td>\n",
       "      <td>6.928361e+02</td>\n",
       "      <td>1.358013e+03</td>\n",
       "      <td>1.112740e+03</td>\n",
       "      <td>6.423705e+02</td>\n",
       "      <td>1.551437e+03</td>\n",
       "      <td>8.563209e+02</td>\n",
       "      <td>1.135954e+03</td>\n",
       "      <td>3.071974e+03</td>\n",
       "      <td>2.626834e+01</td>\n",
       "      <td>2.306265e+02</td>\n",
       "      <td>3.060298e+03</td>\n",
       "      <td>3.042039e+03</td>\n",
       "      <td>7.980961e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>111.0</td>\n",
       "      <td>1.997000e+03</td>\n",
       "      <td>-5.336000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>111.0</td>\n",
       "      <td>2.002000e+03</td>\n",
       "      <td>9.300000e+01</td>\n",
       "      <td>6.100000e+02</td>\n",
       "      <td>1.010000e+00</td>\n",
       "      <td>5.390000e-01</td>\n",
       "      <td>9.450000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>5.973000e+01</td>\n",
       "      <td>1.817000e+01</td>\n",
       "      <td>2.092000e+01</td>\n",
       "      <td>2.158000e+01</td>\n",
       "      <td>9.420000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>111.0</td>\n",
       "      <td>2.008000e+03</td>\n",
       "      <td>1.840000e+02</td>\n",
       "      <td>1.210000e+03</td>\n",
       "      <td>1.285000e+03</td>\n",
       "      <td>1.253000e+00</td>\n",
       "      <td>1.554000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-2.749000e+02</td>\n",
       "      <td>7.660000e+01</td>\n",
       "      <td>2.114000e+01</td>\n",
       "      <td>2.526000e+01</td>\n",
       "      <td>2.556000e+01</td>\n",
       "      <td>9.460000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>111.0</td>\n",
       "      <td>2.013000e+03</td>\n",
       "      <td>2.750000e+02</td>\n",
       "      <td>1.810000e+03</td>\n",
       "      <td>2.187000e+03</td>\n",
       "      <td>1.594000e+03</td>\n",
       "      <td>2.531000e+02</td>\n",
       "      <td>3.541250e+02</td>\n",
       "      <td>1.563000e+01</td>\n",
       "      <td>9.940000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.840000e+01</td>\n",
       "      <td>2.511000e+01</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>9.510000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>111.0</td>\n",
       "      <td>2.139000e+03</td>\n",
       "      <td>6.540000e+02</td>\n",
       "      <td>6.640000e+03</td>\n",
       "      <td>6.996000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>6.836000e+03</td>\n",
       "      <td>1.822660e+05</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>2.802000e+03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             COD          YEAR        DAY365    MINUTE2400   N_WNDVEL_5M  \\\n",
       "count  1138228.0  1.138228e+06  1.138228e+06  1.138228e+06  1.138228e+06   \n",
       "mean       111.0  2.007567e+03  1.834570e+02  1.191838e+03  1.338263e+03   \n",
       "std          0.0  6.320072e+00  1.054385e+02  6.928361e+02  1.358013e+03   \n",
       "min        111.0  1.997000e+03 -5.336000e+01  1.000000e+01 -6.999000e+03   \n",
       "25%        111.0  2.002000e+03  9.300000e+01  6.100000e+02  1.010000e+00   \n",
       "50%        111.0  2.008000e+03  1.840000e+02  1.210000e+03  1.285000e+03   \n",
       "75%        111.0  2.013000e+03  2.750000e+02  1.810000e+03  2.187000e+03   \n",
       "max        111.0  2.139000e+03  6.540000e+02  6.640000e+03  6.996000e+03   \n",
       "\n",
       "        N_WNDVEL_2M  N_WNDDIR_DEGREE  N_INCRAD_10MSR  N_REFRAD_10MSR  \\\n",
       "count  1.138228e+06     1.138228e+06    1.138228e+06    1.138228e+06   \n",
       "mean   8.314822e+02     2.618663e+02    7.067442e+02    2.609247e+02   \n",
       "std    1.112740e+03     6.423705e+02    1.551437e+03    8.563209e+02   \n",
       "min    0.000000e+00     0.000000e+00   -6.999000e+03    0.000000e+00   \n",
       "25%    5.390000e-01     9.450000e+01    0.000000e+00    0.000000e+00   \n",
       "50%    1.253000e+00     1.554000e+02    0.000000e+00    0.000000e+00   \n",
       "75%    1.594000e+03     2.531000e+02    3.541250e+02    1.563000e+01   \n",
       "max    6.999000e+03     6.999000e+03    6.999000e+03    6.999000e+03   \n",
       "\n",
       "       N_PHTFLOW_10MSR  N_NHTFLOW_10MSR  N_AIRMTR_PCT10  N_AIRTMP_MED10  \\\n",
       "count     1.138228e+06     1.138228e+06    1.138227e+06    1.138227e+06   \n",
       "mean      3.430510e+02    -2.136187e+03    7.328247e+01    2.550711e+01   \n",
       "std       1.135954e+03     3.071974e+03    2.626834e+01    2.306265e+02   \n",
       "min       0.000000e+00    -6.999000e+03   -6.999000e+03   -6.999000e+03   \n",
       "25%       0.000000e+00    -6.999000e+03    5.973000e+01    1.817000e+01   \n",
       "50%       0.000000e+00    -2.749000e+02    7.660000e+01    2.114000e+01   \n",
       "75%       9.940000e+01     0.000000e+00    8.840000e+01    2.511000e+01   \n",
       "max       6.999000e+03     0.000000e+00    6.836000e+03    1.822660e+05   \n",
       "\n",
       "       N_GRDTMP_3CM  N_GRDTMP_6CM  N_PRESSION_HPA  \n",
       "count  1.138227e+06  1.138227e+06    1.138227e+06  \n",
       "mean   1.834196e+03  1.804436e+03    9.460472e+02  \n",
       "std    3.060298e+03  3.042039e+03    7.980961e+01  \n",
       "min   -6.999000e+03 -6.999000e+03   -6.999000e+03  \n",
       "25%    2.092000e+01  2.158000e+01    9.420000e+02  \n",
       "50%    2.526000e+01  2.556000e+01    9.460000e+02  \n",
       "75%    6.999000e+03  6.999000e+03    9.510000e+02  \n",
       "max    6.999000e+03  6.999000e+03    2.802000e+03  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we can see that there are a few values inconsistent, like negative value for \"Day's variable\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Counting 0 value for Day variable\n",
    "base[base['DAY365'] <1 ]['YEAR'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-53.36,   0.  ])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking these values\n",
    "base[base['DAY365'] <1 ]['DAY365'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2019.,  2000.])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking which Years tehse inconsistent values exist\n",
    "base[base['DAY365'] <1 ]['YEAR'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also there are values above 366 in Day Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 388.,  654.])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking\n",
    "base[base['DAY365'] > 366 ]['DAY365'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2039.,  2139.])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking years of occurence\n",
    "base[base['DAY365'] > 366 ]['YEAR'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since out datasets are from 1997 to 2017 values above 2017 are also inconsistent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Couting inconsistent values\n",
    "base[base['YEAR']> 2018]['YEAR'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2019.,  2039.,  2126.,  2139.])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking these values\n",
    "base[base['YEAR']> 2018]['YEAR'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other variable with directly notable inconsistent values is \"Hora - Minuto\" that is hour variable, it is limited from 0 to 2400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#counting\n",
    "base[base['MINUTE2400']> 2400]['MINUTE2400'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 6640.])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#looking\n",
    "base[base['MINUTE2400']> 2400]['MINUTE2400'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All these values will be tagged to future treatment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "COD                0\n",
       "YEAR               0\n",
       "DAY365             0\n",
       "MINUTE2400         0\n",
       "N_WNDVEL_5M        0\n",
       "N_WNDVEL_2M        0\n",
       "N_WNDDIR_DEGREE    0\n",
       "N_INCRAD_10MSR     0\n",
       "N_REFRAD_10MSR     0\n",
       "N_PHTFLOW_10MSR    0\n",
       "N_NHTFLOW_10MSR    0\n",
       "N_AIRMTR_PCT10     1\n",
       "N_AIRTMP_MED10     1\n",
       "N_GRDTMP_3CM       1\n",
       "N_GRDTMP_6CM       1\n",
       "N_GRDTMP_UV        1\n",
       "N_PRESSION_HPA     1\n",
       "N_RAIN_SUM10       1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Counting all\n",
    "base.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These values will be also tagged later"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fixing inconsistent values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Function to fix year variable\n",
    "def fix_year(anos):\n",
    "    if(anos in [2019, 2039, 2126, 2139]):\n",
    "        return -999\n",
    "    else:\n",
    "        return anos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base['YEAR'] = base['YEAR'].map(fix_year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Function to fix hour variable\n",
    "def fix_hour(hours):\n",
    "    if(hours == 6640):\n",
    "        return -999\n",
    "    else:\n",
    "        return hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base[\"MINUTE2400\"] = base[\"MINUTE2400\"].map(fix_hour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Function to fix day variable\n",
    "def fix_day(days):\n",
    "    if((days<1) | (days > 366)):\n",
    "        return -999\n",
    "    else:\n",
    "        return days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base[\"DAY365\"] = base[\"DAY365\"].map(fix_day)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base = base.dropna();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1138227, 18)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking data consistency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "COD                0\n",
       "YEAR               0\n",
       "DAY365             0\n",
       "MINUTE2400         0\n",
       "N_WNDVEL_5M        0\n",
       "N_WNDVEL_2M        0\n",
       "N_WNDDIR_DEGREE    0\n",
       "N_INCRAD_10MSR     0\n",
       "N_REFRAD_10MSR     0\n",
       "N_PHTFLOW_10MSR    0\n",
       "N_NHTFLOW_10MSR    0\n",
       "N_AIRMTR_PCT10     0\n",
       "N_AIRTMP_MED10     0\n",
       "N_GRDTMP_3CM       0\n",
       "N_GRDTMP_6CM       0\n",
       "N_GRDTMP_UV        0\n",
       "N_PRESSION_HPA     0\n",
       "N_RAIN_SUM10       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base[(base['DAY365'] <1 ) & (base['DAY365']!=-999)]['YEAR'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base[base['YEAR']> 2018]['YEAR'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base[base['MINUTE2400']> 2400]['MINUTE2400'].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Treatment of variables with non numerical data, where it should be"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vars_ = base.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theres is non numerical data in variable :  N_GRDTMP_UV\n",
      "Theres is non numerical data in variable :  N_RAIN_SUM10\n"
     ]
    }
   ],
   "source": [
    "for v in vars_: \n",
    "    try:\n",
    "        base[v] = base[v].astype(float)\n",
    "    except ValueError:\n",
    "    #Handle the exception\n",
    "        print(\"Theres is non numerical data in variable : \", v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Fixing these values\n",
    "\n",
    "base['N_GRDTMP_UV'] = pd.to_numeric(base['N_GRDTMP_UV'], errors='coerce')\n",
    "base['N_RAIN_SUM10']= pd.to_numeric(base['N_RAIN_SUM10'], errors = 'coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base = base.dropna();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "COD                0\n",
       "YEAR               0\n",
       "DAY365             0\n",
       "MINUTE2400         0\n",
       "N_WNDVEL_5M        0\n",
       "N_WNDVEL_2M        0\n",
       "N_WNDDIR_DEGREE    0\n",
       "N_INCRAD_10MSR     0\n",
       "N_REFRAD_10MSR     0\n",
       "N_PHTFLOW_10MSR    0\n",
       "N_NHTFLOW_10MSR    0\n",
       "N_AIRMTR_PCT10     0\n",
       "N_AIRTMP_MED10     0\n",
       "N_GRDTMP_3CM       0\n",
       "N_GRDTMP_6CM       0\n",
       "N_GRDTMP_UV        0\n",
       "N_PRESSION_HPA     0\n",
       "N_RAIN_SUM10       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "COD                  0\n",
       "YEAR               128\n",
       "DAY365             131\n",
       "MINUTE2400           1\n",
       "N_WNDVEL_5M          0\n",
       "N_WNDVEL_2M          0\n",
       "N_WNDDIR_DEGREE      0\n",
       "N_INCRAD_10MSR       0\n",
       "N_REFRAD_10MSR       0\n",
       "N_PHTFLOW_10MSR      0\n",
       "N_NHTFLOW_10MSR     17\n",
       "N_AIRMTR_PCT10       0\n",
       "N_AIRTMP_MED10       0\n",
       "N_GRDTMP_3CM         0\n",
       "N_GRDTMP_6CM         0\n",
       "N_GRDTMP_UV          0\n",
       "N_PRESSION_HPA       0\n",
       "N_RAIN_SUM10         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((base == -1000) | (base == -999)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clean_base = base[((base != -1000) & (base != -999))]\n",
    "clean_base = clean_base.dropna();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Here we export our main resulted dataset of this code\n",
    "clean_base['YEAR'] = clean_base['YEAR'].astype(int)\n",
    "clean_base.to_csv(\"clean_base111.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "COD                0\n",
       "YEAR               0\n",
       "DAY365             0\n",
       "MINUTE2400         0\n",
       "N_WNDVEL_5M        0\n",
       "N_WNDVEL_2M        0\n",
       "N_WNDDIR_DEGREE    0\n",
       "N_INCRAD_10MSR     0\n",
       "N_REFRAD_10MSR     0\n",
       "N_PHTFLOW_10MSR    0\n",
       "N_NHTFLOW_10MSR    0\n",
       "N_AIRMTR_PCT10     0\n",
       "N_AIRTMP_MED10     0\n",
       "N_GRDTMP_3CM       0\n",
       "N_GRDTMP_6CM       0\n",
       "N_GRDTMP_UV        0\n",
       "N_PRESSION_HPA     0\n",
       "N_RAIN_SUM10       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_base.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1138003, 18)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_base.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00019767568536355196"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - 1138003/1138228"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}