2.1 - [Data Cleaning] Control variables.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\hideki\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2698: DtypeWarning: Columns (15,17) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "# Importing base_111 file\n",
    "base = pd.read_csv(\"base_111.csv\", encoding=\"latin-1\")\n",
    "base2 = pd.read_csv(\"base_222.csv\", encoding=\"latin-1\")\n",
    "base265 = pd.read_csv(\"base_265.csv\", encoding=\"latin-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cod</th>\n",
       "      <th>Ano</th>\n",
       "      <th>Dia Juliano</th>\n",
       "      <th>Hora - minuto</th>\n",
       "      <th>Veloc. Instantânea do Vento a 5 metros (m/s)</th>\n",
       "      <th>Veloc. Instantânea do Vento a 2 metros (m/s)</th>\n",
       "      <th>Direção do Vento no instante da aquisição (°)</th>\n",
       "      <th>Radiação Incidente total de 10 leituras (KW/m2)</th>\n",
       "      <th>Radiação Refletida Total de 10 leituras (KW/m2)</th>\n",
       "      <th>Fluxo Positivo de calor no solo total de 10 leituras (W/m2)</th>\n",
       "      <th>Fluxo negativo de calor no solo . total de 10 leituras (W/m2)</th>\n",
       "      <th>Umidade relativa Média (%)</th>\n",
       "      <th>Temperatura do Ar Média (°C)</th>\n",
       "      <th>Temperatura do solo  1 Média (°C) à 3cm no solo</th>\n",
       "      <th>Temperatura do solo 2Média (°C) à 6cm no solo</th>\n",
       "      <th>Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09</th>\n",
       "      <th>Pressão (hPa)</th>\n",
       "      <th>Chuva total do período (mm)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111</td>\n",
       "      <td>1997</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>1618.00</td>\n",
       "      <td>1115.000</td>\n",
       "      <td>8.32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-443.0</td>\n",
       "      <td>92.3</td>\n",
       "      <td>19.84</td>\n",
       "      <td>21.04</td>\n",
       "      <td>22.68</td>\n",
       "      <td>24.59</td>\n",
       "      <td>943.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>111</td>\n",
       "      <td>1997</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.763</td>\n",
       "      <td>20.59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-443.0</td>\n",
       "      <td>93.1</td>\n",
       "      <td>19.63</td>\n",
       "      <td>20.97</td>\n",
       "      <td>22.65</td>\n",
       "      <td>24.52</td>\n",
       "      <td>943.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cod   Ano  Dia Juliano  Hora - minuto  \\\n",
       "0  111  1997          1.0             10   \n",
       "1  111  1997          1.0             20   \n",
       "\n",
       "   Veloc. Instantânea do Vento a 5 metros (m/s)  \\\n",
       "0                                       1618.00   \n",
       "1                                          0.99   \n",
       "\n",
       "   Veloc. Instantânea do Vento a 2 metros (m/s)  \\\n",
       "0                                      1115.000   \n",
       "1                                         0.763   \n",
       "\n",
       "   Direção do Vento no instante da aquisição (°)  \\\n",
       "0                                           8.32   \n",
       "1                                          20.59   \n",
       "\n",
       "   Radiação Incidente total de 10 leituras (KW/m2)  \\\n",
       "0                                              0.0   \n",
       "1                                              0.0   \n",
       "\n",
       "   Radiação Refletida Total de 10 leituras (KW/m2)  \\\n",
       "0                                              0.0   \n",
       "1                                              0.0   \n",
       "\n",
       "   Fluxo Positivo de calor no solo total de 10 leituras (W/m2)  \\\n",
       "0                                                0.0             \n",
       "1                                                0.0             \n",
       "\n",
       "   Fluxo negativo de calor no solo . total de 10 leituras (W/m2)  \\\n",
       "0                                             -443.0               \n",
       "1                                             -443.0               \n",
       "\n",
       "   Umidade relativa Média (%)  Temperatura do Ar Média (°C)  \\\n",
       "0                        92.3                         19.84   \n",
       "1                        93.1                         19.63   \n",
       "\n",
       "   Temperatura do solo  1 Média (°C) à 3cm no solo  \\\n",
       "0                                            21.04   \n",
       "1                                            20.97   \n",
       "\n",
       "   Temperatura do solo 2Média (°C) à 6cm no solo   \\\n",
       "0                                           22.68   \n",
       "1                                           22.65   \n",
       "\n",
       "  Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09  \\\n",
       "0                                              24.59                  \n",
       "1                                              24.52                  \n",
       "\n",
       "   Pressão (hPa) Chuva total do período (mm)  \n",
       "0          943.0                           0  \n",
       "1          943.0                           0  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cod</th>\n",
       "      <th>Ano</th>\n",
       "      <th>Dia Juliano</th>\n",
       "      <th>Hora - minuto</th>\n",
       "      <th>Veloc. máxima do Vento a 5 metros (m/s)</th>\n",
       "      <th>Veloc. máxima do Vento a 2 metros (m/s)</th>\n",
       "      <th>Direção do Vento no instante da veloc. Máx.(°)</th>\n",
       "      <th>Horário de ocorrência vr máx. veloc. do vento</th>\n",
       "      <th>Radiação Incidente Total do dia (KW/m2)</th>\n",
       "      <th>Radiação Refletida Total do dia (KW/m2)</th>\n",
       "      <th>...</th>\n",
       "      <th>Fluxo Negativo de calor no solo. Total do dia (KW/m2)</th>\n",
       "      <th>Temperatura do Ar  Máxima do dia (°C)</th>\n",
       "      <th>Horário de ocorrência da Temp. Ar Máxima(°C)</th>\n",
       "      <th>Temperatura do Ar Mínima do dia (°C)</th>\n",
       "      <th>Horário de ocorrência da Temp. Ar Minima</th>\n",
       "      <th>Temperatura do Solo 1 Máxima do dia (°C)</th>\n",
       "      <th>Horário de ocorrência da Temp. Solo 1 Máxima.</th>\n",
       "      <th>Temperatura do Solo 1 Mínima do dia (°C)</th>\n",
       "      <th>Horário de ocorrência da Temp. Solo 1 minima</th>\n",
       "      <th>Chuva total do dia (mm)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>222</td>\n",
       "      <td>1997</td>\n",
       "      <td>1</td>\n",
       "      <td>2400</td>\n",
       "      <td>6031.0</td>\n",
       "      <td>2677.0</td>\n",
       "      <td>99.6</td>\n",
       "      <td>1923</td>\n",
       "      <td>270.8</td>\n",
       "      <td>53.81</td>\n",
       "      <td>...</td>\n",
       "      <td>-34.76</td>\n",
       "      <td>30.16</td>\n",
       "      <td>1255</td>\n",
       "      <td>18.50</td>\n",
       "      <td>551.0</td>\n",
       "      <td>34.53</td>\n",
       "      <td>1258</td>\n",
       "      <td>19.15</td>\n",
       "      <td>606.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>222</td>\n",
       "      <td>1997</td>\n",
       "      <td>2</td>\n",
       "      <td>2400</td>\n",
       "      <td>4756.0</td>\n",
       "      <td>1758.0</td>\n",
       "      <td>119.5</td>\n",
       "      <td>45</td>\n",
       "      <td>165.3</td>\n",
       "      <td>27.46</td>\n",
       "      <td>...</td>\n",
       "      <td>-26.86</td>\n",
       "      <td>26.15</td>\n",
       "      <td>1417</td>\n",
       "      <td>19.15</td>\n",
       "      <td>2355.0</td>\n",
       "      <td>27.66</td>\n",
       "      <td>1425</td>\n",
       "      <td>19.65</td>\n",
       "      <td>2352.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cod   Ano  Dia Juliano  Hora - minuto  \\\n",
       "0  222  1997            1           2400   \n",
       "1  222  1997            2           2400   \n",
       "\n",
       "   Veloc. máxima do Vento a 5 metros (m/s)  \\\n",
       "0                                   6031.0   \n",
       "1                                   4756.0   \n",
       "\n",
       "   Veloc. máxima do Vento a 2 metros (m/s)  \\\n",
       "0                                   2677.0   \n",
       "1                                   1758.0   \n",
       "\n",
       "   Direção do Vento no instante da veloc. Máx.(°)  \\\n",
       "0                                            99.6   \n",
       "1                                           119.5   \n",
       "\n",
       "   Horário de ocorrência vr máx. veloc. do vento  \\\n",
       "0                                           1923   \n",
       "1                                             45   \n",
       "\n",
       "   Radiação Incidente Total do dia (KW/m2)  \\\n",
       "0                                    270.8   \n",
       "1                                    165.3   \n",
       "\n",
       "   Radiação Refletida Total do dia (KW/m2)           ...             \\\n",
       "0                                    53.81           ...              \n",
       "1                                    27.46           ...              \n",
       "\n",
       "   Fluxo Negativo de calor no solo. Total do dia (KW/m2)  \\\n",
       "0                                             -34.76       \n",
       "1                                             -26.86       \n",
       "\n",
       "  Temperatura do Ar  Máxima do dia (°C)  \\\n",
       "0                                 30.16   \n",
       "1                                 26.15   \n",
       "\n",
       "  Horário de ocorrência da Temp. Ar Máxima(°C)  \\\n",
       "0                                         1255   \n",
       "1                                         1417   \n",
       "\n",
       "  Temperatura do Ar Mínima do dia (°C)  \\\n",
       "0                                18.50   \n",
       "1                                19.15   \n",
       "\n",
       "   Horário de ocorrência da Temp. Ar Minima  \\\n",
       "0                                     551.0   \n",
       "1                                    2355.0   \n",
       "\n",
       "   Temperatura do Solo 1 Máxima do dia (°C)  \\\n",
       "0                                     34.53   \n",
       "1                                     27.66   \n",
       "\n",
       "  Horário de ocorrência da Temp. Solo 1 Máxima.  \\\n",
       "0                                          1258   \n",
       "1                                          1425   \n",
       "\n",
       "  Temperatura do Solo 1 Mínima do dia (°C)  \\\n",
       "0                                    19.15   \n",
       "1                                    19.65   \n",
       "\n",
       "   Horário de ocorrência da Temp. Solo 1 minima  Chuva total do dia (mm)  \n",
       "0                                         606.0                      0.0  \n",
       "1                                        2352.0                      0.0  \n",
       "\n",
       "[2 rows x 21 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base2.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cod</th>\n",
       "      <th>Ano</th>\n",
       "      <th>Dia Juliano</th>\n",
       "      <th>Hora - minuto</th>\n",
       "      <th>Chuva das 7 h as 7 h</th>\n",
       "      <th>Temperatura do Ar máxima do dia °C</th>\n",
       "      <th>Horário de ocorrência da Temp. Ar máxima</th>\n",
       "      <th>Temperatura do Ar Minima do dia (°C)</th>\n",
       "      <th>Horário de ocorrência da Temp. Ar minima</th>\n",
       "      <th>Umidade máxima</th>\n",
       "      <th>Hora</th>\n",
       "      <th>Umidade mínima</th>\n",
       "      <th>Hora.1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>265</td>\n",
       "      <td>1997</td>\n",
       "      <td>1.0</td>\n",
       "      <td>700</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.11</td>\n",
       "      <td>1356</td>\n",
       "      <td>18.50</td>\n",
       "      <td>551.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>265</td>\n",
       "      <td>1997</td>\n",
       "      <td>2.0</td>\n",
       "      <td>700</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.16</td>\n",
       "      <td>1255</td>\n",
       "      <td>19.21</td>\n",
       "      <td>510.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cod   Ano  Dia Juliano  Hora - minuto  Chuva das 7 h as 7 h  \\\n",
       "0  265  1997          1.0            700                   0.0   \n",
       "1  265  1997          2.0            700                   0.0   \n",
       "\n",
       "   Temperatura do Ar máxima do dia °C  \\\n",
       "0                               31.11   \n",
       "1                               30.16   \n",
       "\n",
       "   Horário de ocorrência da Temp. Ar máxima  \\\n",
       "0                                      1356   \n",
       "1                                      1255   \n",
       "\n",
       "   Temperatura do Ar Minima do dia (°C)  \\\n",
       "0                                 18.50   \n",
       "1                                 19.21   \n",
       "\n",
       "   Horário de ocorrência da Temp. Ar minima  Umidade máxima  Hora  \\\n",
       "0                                     551.0             NaN   NaN   \n",
       "1                                     510.0             NaN   NaN   \n",
       "\n",
       "   Umidade mínima  Hora.1  \n",
       "0             NaN     NaN  \n",
       "1             NaN     NaN  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base265.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Describe data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cod</th>\n",
       "      <th>Ano</th>\n",
       "      <th>Dia Juliano</th>\n",
       "      <th>Hora - minuto</th>\n",
       "      <th>Veloc. Instantânea do Vento a 5 metros (m/s)</th>\n",
       "      <th>Veloc. Instantânea do Vento a 2 metros (m/s)</th>\n",
       "      <th>Direção do Vento no instante da aquisição (°)</th>\n",
       "      <th>Radiação Incidente total de 10 leituras (KW/m2)</th>\n",
       "      <th>Radiação Refletida Total de 10 leituras (KW/m2)</th>\n",
       "      <th>Fluxo Positivo de calor no solo total de 10 leituras (W/m2)</th>\n",
       "      <th>Fluxo negativo de calor no solo . total de 10 leituras (W/m2)</th>\n",
       "      <th>Umidade relativa Média (%)</th>\n",
       "      <th>Temperatura do Ar Média (°C)</th>\n",
       "      <th>Temperatura do solo  1 Média (°C) à 3cm no solo</th>\n",
       "      <th>Temperatura do solo 2Média (°C) à 6cm no solo</th>\n",
       "      <th>Pressão (hPa)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.050445e+06</td>\n",
       "      <td>1.048620e+06</td>\n",
       "      <td>1.048619e+06</td>\n",
       "      <td>1.048619e+06</td>\n",
       "      <td>1.048254e+06</td>\n",
       "      <td>1.048254e+06</td>\n",
       "      <td>1.048254e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.114778e+02</td>\n",
       "      <td>2.006549e+03</td>\n",
       "      <td>1.835618e+02</td>\n",
       "      <td>1.192104e+03</td>\n",
       "      <td>1.472154e+03</td>\n",
       "      <td>9.129840e+02</td>\n",
       "      <td>2.723121e+02</td>\n",
       "      <td>7.622076e+02</td>\n",
       "      <td>2.706437e+02</td>\n",
       "      <td>3.653693e+02</td>\n",
       "      <td>-1.662485e+03</td>\n",
       "      <td>7.404259e+01</td>\n",
       "      <td>2.634531e+01</td>\n",
       "      <td>1.332214e+03</td>\n",
       "      <td>1.296305e+03</td>\n",
       "      <td>9.462274e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.895335e+00</td>\n",
       "      <td>5.762457e+00</td>\n",
       "      <td>1.053855e+02</td>\n",
       "      <td>6.927634e+02</td>\n",
       "      <td>1.353874e+03</td>\n",
       "      <td>1.134761e+03</td>\n",
       "      <td>6.693158e+02</td>\n",
       "      <td>1.611430e+03</td>\n",
       "      <td>8.889303e+02</td>\n",
       "      <td>1.147225e+03</td>\n",
       "      <td>2.796438e+03</td>\n",
       "      <td>2.748618e+01</td>\n",
       "      <td>2.413575e+02</td>\n",
       "      <td>2.725003e+03</td>\n",
       "      <td>2.694402e+03</td>\n",
       "      <td>8.089817e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.110000e+02</td>\n",
       "      <td>1.997000e+03</td>\n",
       "      <td>-5.336000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "      <td>-6.999000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.110000e+02</td>\n",
       "      <td>2.002000e+03</td>\n",
       "      <td>9.300000e+01</td>\n",
       "      <td>6.100000e+02</td>\n",
       "      <td>1.010000e+00</td>\n",
       "      <td>5.390000e-01</td>\n",
       "      <td>9.380000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-5.298000e+02</td>\n",
       "      <td>6.053000e+01</td>\n",
       "      <td>1.813000e+01</td>\n",
       "      <td>2.049000e+01</td>\n",
       "      <td>2.117000e+01</td>\n",
       "      <td>9.430000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.110000e+02</td>\n",
       "      <td>2.007000e+03</td>\n",
       "      <td>1.840000e+02</td>\n",
       "      <td>1.210000e+03</td>\n",
       "      <td>1.481000e+03</td>\n",
       "      <td>1.010000e+00</td>\n",
       "      <td>1.555000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-2.460000e+02</td>\n",
       "      <td>7.750000e+01</td>\n",
       "      <td>2.114000e+01</td>\n",
       "      <td>2.448000e+01</td>\n",
       "      <td>2.493000e+01</td>\n",
       "      <td>9.470000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.110000e+02</td>\n",
       "      <td>2.012000e+03</td>\n",
       "      <td>2.750000e+02</td>\n",
       "      <td>1.810000e+03</td>\n",
       "      <td>2.265000e+03</td>\n",
       "      <td>1.716000e+03</td>\n",
       "      <td>2.576000e+02</td>\n",
       "      <td>4.338000e+02</td>\n",
       "      <td>9.890000e-01</td>\n",
       "      <td>1.788000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.920000e+01</td>\n",
       "      <td>2.510000e+01</td>\n",
       "      <td>3.198000e+01</td>\n",
       "      <td>2.988000e+01</td>\n",
       "      <td>9.520000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.330000e+02</td>\n",
       "      <td>2.139000e+03</td>\n",
       "      <td>6.540000e+02</td>\n",
       "      <td>6.640000e+03</td>\n",
       "      <td>6.996000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.538000e+03</td>\n",
       "      <td>6.836000e+03</td>\n",
       "      <td>1.822660e+05</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>6.999000e+03</td>\n",
       "      <td>2.802000e+03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Cod           Ano   Dia Juliano  Hora - minuto  \\\n",
       "count  1.050445e+06  1.050445e+06  1.050445e+06   1.050445e+06   \n",
       "mean   1.114778e+02  2.006549e+03  1.835618e+02   1.192104e+03   \n",
       "std    9.895335e+00  5.762457e+00  1.053855e+02   6.927634e+02   \n",
       "min    1.110000e+02  1.997000e+03 -5.336000e+01   1.000000e+01   \n",
       "25%    1.110000e+02  2.002000e+03  9.300000e+01   6.100000e+02   \n",
       "50%    1.110000e+02  2.007000e+03  1.840000e+02   1.210000e+03   \n",
       "75%    1.110000e+02  2.012000e+03  2.750000e+02   1.810000e+03   \n",
       "max    3.330000e+02  2.139000e+03  6.540000e+02   6.640000e+03   \n",
       "\n",
       "       Veloc. Instantânea do Vento a 5 metros (m/s)  \\\n",
       "count                                  1.050445e+06   \n",
       "mean                                   1.472154e+03   \n",
       "std                                    1.353874e+03   \n",
       "min                                   -6.999000e+03   \n",
       "25%                                    1.010000e+00   \n",
       "50%                                    1.481000e+03   \n",
       "75%                                    2.265000e+03   \n",
       "max                                    6.996000e+03   \n",
       "\n",
       "       Veloc. Instantânea do Vento a 2 metros (m/s)  \\\n",
       "count                                  1.050445e+06   \n",
       "mean                                   9.129840e+02   \n",
       "std                                    1.134761e+03   \n",
       "min                                    0.000000e+00   \n",
       "25%                                    5.390000e-01   \n",
       "50%                                    1.010000e+00   \n",
       "75%                                    1.716000e+03   \n",
       "max                                    6.999000e+03   \n",
       "\n",
       "       Direção do Vento no instante da aquisição (°)  \\\n",
       "count                                   1.050445e+06   \n",
       "mean                                    2.723121e+02   \n",
       "std                                     6.693158e+02   \n",
       "min                                     0.000000e+00   \n",
       "25%                                     9.380000e+01   \n",
       "50%                                     1.555000e+02   \n",
       "75%                                     2.576000e+02   \n",
       "max                                     6.999000e+03   \n",
       "\n",
       "       Radiação Incidente total de 10 leituras (KW/m2)  \\\n",
       "count                                     1.050445e+06   \n",
       "mean                                      7.622076e+02   \n",
       "std                                       1.611430e+03   \n",
       "min                                      -6.999000e+03   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       0.000000e+00   \n",
       "75%                                       4.338000e+02   \n",
       "max                                       6.999000e+03   \n",
       "\n",
       "       Radiação Refletida Total de 10 leituras (KW/m2)  \\\n",
       "count                                     1.050445e+06   \n",
       "mean                                      2.706437e+02   \n",
       "std                                       8.889303e+02   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       0.000000e+00   \n",
       "75%                                       9.890000e-01   \n",
       "max                                       6.999000e+03   \n",
       "\n",
       "       Fluxo Positivo de calor no solo total de 10 leituras (W/m2)  \\\n",
       "count                                       1.050445e+06             \n",
       "mean                                        3.653693e+02             \n",
       "std                                         1.147225e+03             \n",
       "min                                         0.000000e+00             \n",
       "25%                                         0.000000e+00             \n",
       "50%                                         0.000000e+00             \n",
       "75%                                         1.788000e+02             \n",
       "max                                         6.999000e+03             \n",
       "\n",
       "       Fluxo negativo de calor no solo . total de 10 leituras (W/m2)  \\\n",
       "count                                       1.048620e+06               \n",
       "mean                                       -1.662485e+03               \n",
       "std                                         2.796438e+03               \n",
       "min                                        -6.999000e+03               \n",
       "25%                                        -5.298000e+02               \n",
       "50%                                        -2.460000e+02               \n",
       "75%                                         0.000000e+00               \n",
       "max                                         6.538000e+03               \n",
       "\n",
       "       Umidade relativa Média (%)  Temperatura do Ar Média (°C)  \\\n",
       "count                1.048619e+06                  1.048619e+06   \n",
       "mean                 7.404259e+01                  2.634531e+01   \n",
       "std                  2.748618e+01                  2.413575e+02   \n",
       "min                 -6.999000e+03                 -6.999000e+03   \n",
       "25%                  6.053000e+01                  1.813000e+01   \n",
       "50%                  7.750000e+01                  2.114000e+01   \n",
       "75%                  8.920000e+01                  2.510000e+01   \n",
       "max                  6.836000e+03                  1.822660e+05   \n",
       "\n",
       "       Temperatura do solo  1 Média (°C) à 3cm no solo  \\\n",
       "count                                     1.048254e+06   \n",
       "mean                                      1.332214e+03   \n",
       "std                                       2.725003e+03   \n",
       "min                                      -6.999000e+03   \n",
       "25%                                       2.049000e+01   \n",
       "50%                                       2.448000e+01   \n",
       "75%                                       3.198000e+01   \n",
       "max                                       6.999000e+03   \n",
       "\n",
       "       Temperatura do solo 2Média (°C) à 6cm no solo   Pressão (hPa)  \n",
       "count                                    1.048254e+06   1.048254e+06  \n",
       "mean                                     1.296305e+03   9.462274e+02  \n",
       "std                                      2.694402e+03   8.089817e+01  \n",
       "min                                     -6.999000e+03  -6.999000e+03  \n",
       "25%                                      2.117000e+01   9.430000e+02  \n",
       "50%                                      2.493000e+01   9.470000e+02  \n",
       "75%                                      2.988000e+01   9.520000e+02  \n",
       "max                                      6.999000e+03   2.802000e+03  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we can see that there are a few values inconsistent, like negative value for \"Day's variable\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Counting 0 value for Day variable\n",
    "base[base['Dia Juliano'] <1 ]['Ano'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-53.36,   0.  ])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking these values\n",
    "base[base['Dia Juliano'] <1 ]['Dia Juliano'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2019, 2000], dtype=int64)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking which Years tehse inconsistent values exist\n",
    "base[base['Dia Juliano'] <1 ]['Ano'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also there are values above 366 in Day Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 388.,  654.])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking\n",
    "base[base['Dia Juliano'] > 366 ]['Dia Juliano'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2039, 2139], dtype=int64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking years of occurence\n",
    "base[base['Dia Juliano'] > 366 ]['Ano'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since out datasets are from 1997 to 2017 values above 2017 are also inconsistent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Couting inconsistent values\n",
    "base[base['Ano']> 2017]['Ano'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2019, 2039, 2126, 2139], dtype=int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking these values\n",
    "base[base['Ano']> 2017]['Ano'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other variable with directly notable inconsistent values is \"Hora - Minuto\" that is hour variable, it is limited from 0 to 2400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#counting\n",
    "base[base['Hora - minuto']> 2400]['Hora - minuto'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6640], dtype=int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#looking\n",
    "base[base['Hora - minuto']> 2400]['Hora - minuto'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All these values will be tagged to future treatment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Cod                                                                     0\n",
       "Ano                                                                     0\n",
       "Dia Juliano                                                             0\n",
       "Hora - minuto                                                           0\n",
       "Veloc. Instantânea do Vento a 5 metros (m/s)                            0\n",
       "Veloc. Instantânea do Vento a 2 metros (m/s)                            0\n",
       "Direção do Vento no instante da aquisição (°)                           0\n",
       "Radiação Incidente total de 10 leituras (KW/m2)                         0\n",
       "Radiação Refletida Total de 10 leituras (KW/m2)                         0\n",
       "Fluxo Positivo de calor no solo total de 10 leituras (W/m2)             0\n",
       "Fluxo negativo de calor no solo . total de 10 leituras (W/m2)        1825\n",
       "Umidade relativa Média (%)                                           1826\n",
       "Temperatura do Ar Média (°C)                                         1826\n",
       "Temperatura do solo  1 Média (°C) à 3cm no solo                      2191\n",
       "Temperatura do solo 2Média (°C) à 6cm no solo                        2191\n",
       "Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09    2191\n",
       "Pressão (hPa)                                                        2191\n",
       "Chuva total do período (mm)                                          2191\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16432"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Counting all\n",
    "base.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These values will be also tagged later"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fixing inconsistent values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Function to fix year variable\n",
    "def fix_year(anos):\n",
    "    if(anos in [2019, 2039, 2126, 2139]):\n",
    "        return -999\n",
    "    else:\n",
    "        return anos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base['Ano'] = base['Ano'].map(fix_year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Function to fix hour variable\n",
    "def fix_hour(hours):\n",
    "    if(hours == 6640):\n",
    "        return -999\n",
    "    else:\n",
    "        return hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base[\"Hora - minuto\"] = base[\"Hora - minuto\"].map(fix_hour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Function to fix day variable\n",
    "def fix_day(days):\n",
    "    if((days<1) | (days > 366)):\n",
    "        return -999\n",
    "    else:\n",
    "        return days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base[\"Dia Juliano\"] = base[\"Dia Juliano\"].map(fix_day)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base = base.dropna();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1048254, 18)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking data consistency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Cod                                                                  0\n",
       "Ano                                                                  0\n",
       "Dia Juliano                                                          0\n",
       "Hora - minuto                                                        0\n",
       "Veloc. Instantânea do Vento a 5 metros (m/s)                         0\n",
       "Veloc. Instantânea do Vento a 2 metros (m/s)                         0\n",
       "Direção do Vento no instante da aquisição (°)                        0\n",
       "Radiação Incidente total de 10 leituras (KW/m2)                      0\n",
       "Radiação Refletida Total de 10 leituras (KW/m2)                      0\n",
       "Fluxo Positivo de calor no solo total de 10 leituras (W/m2)          0\n",
       "Fluxo negativo de calor no solo . total de 10 leituras (W/m2)        0\n",
       "Umidade relativa Média (%)                                           0\n",
       "Temperatura do Ar Média (°C)                                         0\n",
       "Temperatura do solo  1 Média (°C) à 3cm no solo                      0\n",
       "Temperatura do solo 2Média (°C) à 6cm no solo                        0\n",
       "Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09    0\n",
       "Pressão (hPa)                                                        0\n",
       "Chuva total do período (mm)                                          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base[(base['Dia Juliano'] <1 ) & (base['Dia Juliano']!=-999)]['Ano'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base[base['Ano']> 2017]['Ano'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base[base['Hora - minuto']> 2400]['Hora - minuto'].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Treatment of variables with non numerical data, where it should be"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vars_ = base.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theres is non numerical data in variable :  Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09\n",
      "Theres is non numerical data in variable :  Chuva total do período (mm)\n"
     ]
    }
   ],
   "source": [
    "for v in vars_: \n",
    "    try:\n",
    "        base[v] = base[v].astype(float)\n",
    "    except ValueError:\n",
    "    #Handle the exception\n",
    "        print(\"Theres is non numerical data in variable : \", v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Fixing these values\n",
    "\n",
    "base['Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09'] = pd.to_numeric(base['Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09'], errors='coerce')\n",
    "base['Chuva total do período (mm)']= pd.to_numeric(base['Chuva total do período (mm)'], errors = 'coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "base = base.dropna();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Cod                                                                  0\n",
       "Ano                                                                  0\n",
       "Dia Juliano                                                          0\n",
       "Hora - minuto                                                        0\n",
       "Veloc. Instantânea do Vento a 5 metros (m/s)                         0\n",
       "Veloc. Instantânea do Vento a 2 metros (m/s)                         0\n",
       "Direção do Vento no instante da aquisição (°)                        0\n",
       "Radiação Incidente total de 10 leituras (KW/m2)                      0\n",
       "Radiação Refletida Total de 10 leituras (KW/m2)                      0\n",
       "Fluxo Positivo de calor no solo total de 10 leituras (W/m2)          0\n",
       "Fluxo negativo de calor no solo . total de 10 leituras (W/m2)        0\n",
       "Umidade relativa Média (%)                                           0\n",
       "Temperatura do Ar Média (°C)                                         0\n",
       "Temperatura do solo  1 Média (°C) à 3cm no solo                      0\n",
       "Temperatura do solo 2Média (°C) à 6cm no solo                        0\n",
       "Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09    0\n",
       "Pressão (hPa)                                                        0\n",
       "Chuva total do período (mm)                                          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Cod                                                                    0\n",
       "Ano                                                                  128\n",
       "Dia Juliano                                                          131\n",
       "Hora - minuto                                                          1\n",
       "Veloc. Instantânea do Vento a 5 metros (m/s)                           0\n",
       "Veloc. Instantânea do Vento a 2 metros (m/s)                           0\n",
       "Direção do Vento no instante da aquisição (°)                          0\n",
       "Radiação Incidente total de 10 leituras (KW/m2)                        0\n",
       "Radiação Refletida Total de 10 leituras (KW/m2)                        0\n",
       "Fluxo Positivo de calor no solo total de 10 leituras (W/m2)            0\n",
       "Fluxo negativo de calor no solo . total de 10 leituras (W/m2)         20\n",
       "Umidade relativa Média (%)                                             0\n",
       "Temperatura do Ar Média (°C)                                           0\n",
       "Temperatura do solo  1 Média (°C) à 3cm no solo                        0\n",
       "Temperatura do solo 2Média (°C) à 6cm no solo                          0\n",
       "Temperatura do solo 3Média (°C) Colocado sensor de UV em 31/07/09      0\n",
       "Pressão (hPa)                                                          0\n",
       "Chuva total do período (mm)                                            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((base == -1000) | (base == -999)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clean_base = base[((base != -1000) & (base != -999))]\n",
    "clean_base = clean_base.dropna();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Here we export our main resulted dataset of this code\n",
    "clean_base.to_csv(\"clean_base111.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}