Read_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read meteorological data\n",
    "Folder with [original data](https://drive.google.com/drive/folders/1me2IpDY3om6IRKv_WMT5W6Qw0r9DI74C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import zipfile\n",
    "import pandas as pd\n",
    "import glob\n",
    "import keplergl\n",
    "import geopandas as gpd\n",
    "import matplotlib.pyplot as plt\n",
    "import csv\n",
    "from datetime import datetime\n",
    "import re\n",
    "import numpy as np\n",
    "from pyproj import Proj\n",
    "%run Meteo_utils.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.DS_Store',\n",
       " 'Estaciones_meteorologicas.csv',\n",
       " 'Inventario Estaciones Meteo.xls',\n",
       " '.ipynb_checkpoints',\n",
       " 'PROCESSED',\n",
       " 'RAW']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Automate data extraction from Drive downloads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW'\n",
    "ziplist = glob.glob(f'{path}/*.zip')\n",
    "#ziplist = glob.glob(f'data/*.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/14-HGPT-MT-09.- Estacion Calamaca Convenio-20210716T093623Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/25-INAMHI-M0126.- Patate-20210716T093732Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/23-INAMHI-M0128.- Colegio P. F. Cevallos-20210716T093721Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/15-HGPT-MT-12.- Estacion Pampas de Salasaca-20210716T093632Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/17-HGPT-MT-13.- Estacion Tasinteo-20210716T093648Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/17-HGPT-MT-13.- Estacion Tasinteo-20210716T093648Z-002.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/09-HGPT-MT-02.- Estacion Colegio A. J. Sucre-20210625T055503Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/27-INAMHI-M1243.- Cunchibamba-20210716T093750Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/20-Querochaca-20210716T093659Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/16-HGPT-MT-10.- Estacion Mula Corral-20210716T093641Z-001.zip']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ziplist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['14-HGPT-MT-09.- Estacion Calamaca Convenio/2014-07-25/Calamaca_5min_20140804_subir.csv',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/2013-05-07/Calamaca Convenio_1hora_20130515.xls',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/2014-07-25/Calamaca_5min_20140804.csv',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/2016-04-21/8310_5min_20161024_subir.csv',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/2014-11-17/Calamaca_5min_20141117_subir.csv',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/Calamaca Convenio Todo.csv',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/2013-05-07/Calamaca Convenio_1hora_20130515.csv',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/2015-07-03/8310_5min_20150715_subir.csv',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/2016-01-21/8310_5min_20160125_subir.csv',\n",
       " '14-HGPT-MT-09.- Estacion Calamaca Convenio/2013-08-15/Calamaca_5min_20130821.csv']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zf = zipfile.ZipFile(f'{ziplist[folder]}')\n",
    "zf.namelist()[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "### exclude macos files (only if copying data if using Drive directly these don't appear)\n",
    "r = re.compile('^__MACOSX') # only csv files\n",
    "exclude = list(filter(r.match, zf.namelist()[0:10])) \n",
    "clean = list(set(zf.namelist()[0:10])-set(exclude))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_by_date = order_meteo_zip(path, folder =folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['27-INAMHI-M1243.- Cunchibamba/._Ordenado.csv',\n",
       " '27-INAMHI-M1243.- Cunchibamba/Ordenado.csv']"
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_by_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success sep \";\", latin encoding\n",
      "file =14-HGPT-MT-09.- Estacion Calamaca Convenio/2013-05-07/Calamaca Convenio_1hora_20130515.csv, cols = 7\n"
     ]
    }
   ],
   "source": [
    "df = read_meteo_csv(path =path,folder = folder,file = files_by_date[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Variable</th>\n",
       "      <th>Value</th>\n",
       "      <th>Type</th>\n",
       "      <th>Freq</th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03/03/2013</td>\n",
       "      <td>02:00:00</td>\n",
       "      <td>WindMnDirUnit</td>\n",
       "      <td>171.73</td>\n",
       "      <td>G</td>\n",
       "      <td>1hora</td>\n",
       "      <td>14-HGPT-MT-09.- Estacion Calamaca Convenio/201...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03/03/2013</td>\n",
       "      <td>02:00:00</td>\n",
       "      <td>WindMinSpdSclr</td>\n",
       "      <td>0.21</td>\n",
       "      <td>G</td>\n",
       "      <td>1hora</td>\n",
       "      <td>14-HGPT-MT-09.- Estacion Calamaca Convenio/201...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>03/03/2013</td>\n",
       "      <td>02:00:00</td>\n",
       "      <td>WindMaxSpdSclr</td>\n",
       "      <td>2.94</td>\n",
       "      <td>G</td>\n",
       "      <td>1hora</td>\n",
       "      <td>14-HGPT-MT-09.- Estacion Calamaca Convenio/201...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>03/03/2013</td>\n",
       "      <td>02:00:00</td>\n",
       "      <td>WindMaxDir</td>\n",
       "      <td>174.47</td>\n",
       "      <td>G</td>\n",
       "      <td>1hora</td>\n",
       "      <td>14-HGPT-MT-09.- Estacion Calamaca Convenio/201...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>03/03/2013</td>\n",
       "      <td>03:00:00</td>\n",
       "      <td>BAT</td>\n",
       "      <td>12.91</td>\n",
       "      <td>G</td>\n",
       "      <td>1hora</td>\n",
       "      <td>14-HGPT-MT-09.- Estacion Calamaca Convenio/201...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date      Time        Variable   Value Type   Freq  \\\n",
       "1  03/03/2013  02:00:00   WindMnDirUnit  171.73    G  1hora   \n",
       "2  03/03/2013  02:00:00  WindMinSpdSclr    0.21    G  1hora   \n",
       "3  03/03/2013  02:00:00  WindMaxSpdSclr    2.94    G  1hora   \n",
       "4  03/03/2013  02:00:00      WindMaxDir  174.47    G  1hora   \n",
       "5  03/03/2013  03:00:00             BAT   12.91    G  1hora   \n",
       "\n",
       "                                           file_name  \n",
       "1  14-HGPT-MT-09.- Estacion Calamaca Convenio/201...  \n",
       "2  14-HGPT-MT-09.- Estacion Calamaca Convenio/201...  \n",
       "3  14-HGPT-MT-09.- Estacion Calamaca Convenio/201...  \n",
       "4  14-HGPT-MT-09.- Estacion Calamaca Convenio/201...  \n",
       "5  14-HGPT-MT-09.- Estacion Calamaca Convenio/201...  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CalamacaConvenio_1hora_20130515.csv'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_by_date[0].split('/')[-1].replace(\" \",\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success sep \";\", latin encoding\n",
      "file =24-INAMHI-M0380.- HUambalo/2015/julio_diciembre.csv, cols = 4\n",
      "Success sep \";\", latin encoding\n",
      "file =24-INAMHI-M0380.- HUambalo/2016/Enero_Abril.csv, cols = 4\n",
      "Success sep \";\", latin encoding\n",
      "file =24-INAMHI-M0380.- HUambalo/Mayo-Octubre_subir.csv, cols = 4\n",
      "Success sep \";\", latin encoding\n",
      "file =24-INAMHI-M0380.- HUambalo/2015/Julio- Octubre_subir.csv, cols = 4\n",
      "Success sep \";\", latin encoding\n",
      "file =24-INAMHI-M0380.- HUambalo/2013/2013_subir.csv, cols = 4\n",
      "Success sep \";\", latin encoding\n",
      "file =24-INAMHI-M0380.- HUambalo/2014/2014_huambalo_subir.csv, cols = 4\n",
      "Success sep \";\", latin encoding\n",
      "file =24-INAMHI-M0380.- HUambalo/2015/Enero_Julio.csv, cols = 4\n"
     ]
    }
   ],
   "source": [
    "station = files_by_date[0].split('/')[0].replace(\" \",\"\").replace(\".\",\"\")\n",
    "all_var = get_unique_variables(files_by_date,path =path,folder = folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Variable names:\n",
    "# BAT  is  battery\n",
    "# Not very clear: 'H_M Min','WindMnSpdSclr','WindMnSpdS','WindMnDirUnit'\n",
    "\n",
    "replace_values ={\n",
    "# Temperature\n",
    "    'ATAvg' : ['AT1HrAvg', 'AT5minAvg', 'ATAvg','Temperatura'],\n",
    "    'ATMin' : ['AT1HrMin', 'AT5minMin','T_Min', 'T Min'],\n",
    "    'ATMax' : ['AT1HrMax','AT5minMax', 'T_Max', 'T Max'],\n",
    " \n",
    " # Relative Humidity\n",
    "    'RHAvg' : ['Humedad','RH5minAvg','RHAvg','RelHumidAvg', 'Humedad R', 'RH5m2015'],\n",
    "    'RHMin' : ['H_Min','RH5minMin','RelHumidMin','H_M Min'],\n",
    "    'RHMax' : ['H_Max', 'RH5minMax', 'RelHumidMax','H_R Max'],\n",
    "\n",
    " # Wind Speed\n",
    "    'WSAvg' : ['Velocidad','WS5minAvg','WSAvg','WindMnSpdSclr','WindMnSpdS'],\n",
    "    'WSMin' : ['WindMinSpdSclr','WS5minMin'],\n",
    "    'WSMax' : ['WindMaxSpdSclr','WS5minMax'],\n",
    "\n",
    " # Wind Direction\n",
    "    'WDAvg' : ['Direccion', 'WDAvg','WD5minAvg','WindMnDirUnit','Dirección'],\n",
    "    'WDMin' : ['WD5minMin'],\n",
    "    'WDMax' : ['WD5minMax','WindMaxDir'],\n",
    "\n",
    " # Rain\n",
    "    'PAvg' : ['Precipitacion','TB1hrAcc','TB1minAcc','TB5minAcc','Precipitación']\n",
    "}\n",
    "\n",
    "var_list = ['ATAvg','ATMin','ATMax','RHAvg','RHMin','RHMax', 'WSAvg','WSMin','WSMax','WDAvg','WDMin', 'WDMax','PAvg']\n",
    "#sorted(var_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success sep \";\", latin encoding\n",
      "file =24-INAMHI-M0380.- HUambalo/2015/julio_diciembre.csv, cols = 4\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Date_Time    datetime64[ns]\n",
       "Type                float64\n",
       "Freq                 object\n",
       "file_name           float64\n",
       "ATAvg               float64\n",
       "ATMax               float64\n",
       "ATMin               float64\n",
       "BAT                 float64\n",
       "PAvg                float64\n",
       "RHAvg               float64\n",
       "RHMax               float64\n",
       "RHMin               float64\n",
       "TB1hrAcc            float64\n",
       "TB1minAcc           float64\n",
       "TB5minAcc           float64\n",
       "WDAvg               float64\n",
       "WDMax               float64\n",
       "WDMin               float64\n",
       "WSAvg               float64\n",
       "WSMax               float64\n",
       "WSMin               float64\n",
       "st_name             float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = read_meteo_csv(path =path,folder = 0,file = files_by_date[0])\n",
    "df2 = reformat_df(df=df, replace_values=replace_values)\n",
    "df2.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(files_by_date)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process files in loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run Meteo_utils.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Variable names:\n",
    "# BAT  is  battery\n",
    "# Not very clear: 'H_M Min','WindMnSpdSclr','WindMnSpdS','WindMnDirUnit'\n",
    "replace_values ={\n",
    "# Temperature\n",
    "    'ATAvg' :['AT1HrAvg', 'AT5minAvg', 'ATAvg','Temperatura','Temperatuta','TEMPERATURA AIRE Â°C PROM'],\n",
    "    'ATMin' : ['AT1HrMin', 'AT5minMin','T_Min', 'T Min','TEMPERATURA AIRE Â°C MIN'],\n",
    "    'ATMax' : ['AT1HrMax','AT5minMax', 'T_Max', 'T Max','TEMPERATURA AIRE Â°C MAX'],\n",
    " \n",
    " # Relative Humidity\n",
    "    'RHAvg' : ['Humedad','RH5minAvg','RHAvg','RelHumidAvg', 'Humedad R', 'RH5m2015','HUMEDAD RELATIVA DEL AIRE % PROM'],\n",
    "    'RHMin' : ['H_Min','RH5minMin','RelHumidMin','H_M Min','HUMEDAD RELATIVA DEL AIRE % MIN'],\n",
    "    'RHMax' : ['H_Max', 'RH5minMax', 'RelHumidMax','H_R Max','HUMEDAD RELATIVA DEL AIRE % MAX'],\n",
    "\n",
    " # Wind Speed\n",
    "    'WSAvg' : ['Velocidad','WS5minAvg','WSAvg','WindMnSpdSclr','WindMnSpdS','VIENTO VELOCIDAD m/s INST'],\n",
    "    'WSMin' : ['WindMinSpdSclr','WS5minMin'],\n",
    "    'WSMax' : ['WindMaxSpdSclr','WS5minMax'],\n",
    "\n",
    " # Wind Direction\n",
    "    'WDAvg' : ['Direccion', 'WDAvg','WD5minAvg','WindMnDirUnit','Dirección','VIENTO DIRECCION Â° INST'],\n",
    "    'WDMin' : ['WD5minMin'],\n",
    "    'WDMax' : ['WD5minMax','WindMaxDir'],\n",
    "\n",
    " # Rain\n",
    "    'PAvg' : ['Precipitacion','TB1hrAcc','TB1minAcc','TB5minAcc','Precipitación','PRECIPITACION mm SUM']\n",
    "}\n",
    "\n",
    "var_list = ['ATAvg','ATMin','ATMax','RHAvg','RHMin','RHMax', 'WSAvg','WSMin','WSMax','WDAvg','WDMin', 'WDMax','PAvg']\n",
    "#sorted(var_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/14-HGPT-MT-09.- Estacion Calamaca Convenio-20210716T093623Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/25-INAMHI-M0126.- Patate-20210716T093732Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/23-INAMHI-M0128.- Colegio P. F. Cevallos-20210716T093721Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/15-HGPT-MT-12.- Estacion Pampas de Salasaca-20210716T093632Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/17-HGPT-MT-13.- Estacion Tasinteo-20210716T093648Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/17-HGPT-MT-13.- Estacion Tasinteo-20210716T093648Z-002.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/09-HGPT-MT-02.- Estacion Colegio A. J. Sucre-20210625T055503Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/27-INAMHI-M1243.- Cunchibamba-20210716T093750Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/20-Querochaca-20210716T093659Z-001.zip',\n",
       " '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW/16-HGPT-MT-10.- Estacion Mula Corral-20210716T093641Z-001.zip']"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW'\n",
    "ziplist = glob.glob(f'{path}/*.zip')\n",
    "ziplist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4)"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file, folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-06-19/MULA CORRAL_5MIN_20180620.csv, cols = 7\n",
      "51/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-06-19/MULA CORRAL_5MIN_20180620 - copia.csv, cols = 7\n",
      "52/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-08-22/MULA CORRAL_5MIN_20180822.csv, cols = 7\n",
      "53/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-08-22/MULA CORRAL_1MIN_20180823.csv, cols = 7\n",
      "54/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-09-27/MULA CORRAL_5MIN_201809270.csv, cols = 7\n",
      "55/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-09-27/MULA CORRAL_1MIN_201809270.csv, cols = 7\n",
      "56/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-12-27/MULA CORRAL_5MIN_20181227.csv, cols = 7\n",
      "57/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2019-04-10/MULA CORRAL_5MIN_20190410.csv, cols = 7\n",
      "58/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2019-07-01/MULA CORRAL_5MIN_20190701.csv, cols = 7\n",
      "59/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2019-08-16/MULA CORRAL_5MIN_20190816.csv, cols = 7\n"
     ]
    }
   ],
   "source": [
    "path = '/Users/tamarahuete/Documents/Github_repos/TFM21/data'\n",
    "folder = 9\n",
    "files_by_date = order_meteo_zip(path=f'{path}/RAW/', folder =folder)\n",
    "#get_unique_variables(files_by_date,path =f'{path}/RAW/',folder = folder)\n",
    "\n",
    "for file in range(50, len(files_by_date)):\n",
    "        print(f'{file}/{len(files_by_date)}')\n",
    "        df = read_meteo_csv(path =f'{path}/RAW',folder = folder,file = files_by_date[file])    \n",
    "        df2 = reformat_df(df=df, replace_values=replace_values)\n",
    "        master_df = master_df.append(df2)\n",
    "        \n",
    "export_name = f'{files_by_date[0].split(\"/\")[0].replace(\" \",\"\").replace(\".\",\"\")}'\n",
    "repeat = list(set([export_name +'.csv']) & set(os.listdir(f'{path}/PROCESSED')))\n",
    "if len(repeat)>0:\n",
    "    master_df.to_csv(f'{path}/PROCESSED/{export_name}_{len(repeat)+1}.csv')\n",
    "else :\n",
    "    master_df.to_csv(f'{path}/PROCESSED/{export_name}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PAvg</th>\n",
       "      <th>Freq</th>\n",
       "      <th>file_name</th>\n",
       "      <th>Date_Time</th>\n",
       "      <th>Type</th>\n",
       "      <th>WSMin</th>\n",
       "      <th>RHMin</th>\n",
       "      <th>WDMax</th>\n",
       "      <th>WSMax</th>\n",
       "      <th>ATAvg</th>\n",
       "      <th>RHMax</th>\n",
       "      <th>ATMax</th>\n",
       "      <th>ATMin</th>\n",
       "      <th>WDAvg</th>\n",
       "      <th>WDMin</th>\n",
       "      <th>WSAvg</th>\n",
       "      <th>RHAvg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5,10</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-01-01 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5,60</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-02-01 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19,20</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-03-01 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0,00</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-04-01 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5,50</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-05-01 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>1,00</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-12-27 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>2,20</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-12-28 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>1,20</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-12-29 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>0,00</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-12-30 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>0,00</td>\n",
       "      <td>1day</td>\n",
       "      <td>27-INAMHI-M1243.- Cunchibamba/Ordenado.csv</td>\n",
       "      <td>2013-12-31 12:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>365 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      PAvg  Freq                                   file_name  \\\n",
       "0     5,10  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "1     5,60  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "2    19,20  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "3     0,00  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "4     5,50  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "..     ...   ...                                         ...   \n",
       "360   1,00  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "361   2,20  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "362   1,20  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "363   0,00  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "364   0,00  1day  27-INAMHI-M1243.- Cunchibamba/Ordenado.csv   \n",
       "\n",
       "              Date_Time  Type  WSMin  RHMin  WDMax  WSMax  ATAvg  RHMax  \\\n",
       "0   2013-01-01 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "1   2013-02-01 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "2   2013-03-01 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "3   2013-04-01 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "4   2013-05-01 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "..                  ...   ...    ...    ...    ...    ...    ...    ...   \n",
       "360 2013-12-27 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "361 2013-12-28 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "362 2013-12-29 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "363 2013-12-30 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "364 2013-12-31 12:00:00   NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "\n",
       "     ATMax  ATMin  WDAvg  WDMin  WSAvg  RHAvg  \n",
       "0      NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "1      NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "2      NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "3      NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "4      NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "..     ...    ...    ...    ...    ...    ...  \n",
       "360    NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "361    NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "362    NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "363    NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "364    NaN    NaN    NaN    NaN    NaN    NaN  \n",
       "\n",
       "[365 rows x 17 columns]"
      ]
     },
     "execution_count": 313,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['20-Querochaca/2016/M0258 QUEROCHACA 2016.csv']"
      ]
     },
     "execution_count": 380,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_by_date = order_meteo_zip(path=f'{path}/RAW/', folder =8,only_csv=False)\n",
    "files_by_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2013-03-19/Mula Corral_1hora_20130326.csv, cols = 7\n",
      "1/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2013-05-07/Mula Corral_1hora_20130515.csv, cols = 7\n",
      "2/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2013-06-13/Mula Corral_5min_20130625.csv, cols = 7\n",
      "3/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2013-10-03/Mula Corral_5min_20131007.csv, cols = 7\n",
      "4/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-01-04/Mula Corral_5min_20140116_subir.csv, cols = 8\n",
      "5/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-01-04/Mula Corral_5min_20140116.csv, cols = 7\n",
      "6/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-04-08/Mula Corral_5min_20140422_subir.csv, cols = 8\n",
      "7/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-04-08/Mula Corral_5min_20140422.csv, cols = 7\n",
      "8/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-06-30/Mula Corral_5min_20140708_subir.csv, cols = 8\n",
      "9/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-06-30/Mula Corral_5min_20140708.csv, cols = 7\n",
      "10/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-07-25/Mula_Corral_5min_20140804.csv, cols = 12\n",
      "11/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-08-26/Mula Corral_5min_20140826_subir.csv, cols = 8\n",
      "12/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-08-26/Mula Corral_5min_20140826.csv, cols = 7\n",
      "13/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-10-20/Mula Corral_5min_20141021_subir.csv, cols = 8\n",
      "14/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-10-20/Mula Corral_5min_20141021.csv, cols = 7\n",
      "15/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-11-24/Mula Corral_5min_20141128_subir.csv, cols = 8\n",
      "16/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2014-11-24/Mula Corral_5min_20141128.csv, cols = 7\n",
      "17/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-02-11/MULA CORRAL_5MIN_20150211_subir.csv, cols = 8\n",
      "18/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-02-11/MULA CORRAL_5MIN_20150211.csv, cols = 7\n",
      "19/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-04-22/CALAMACA MULA CORRAL_5MIN_20150422_subir.csv, cols = 8\n",
      "20/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-04-22/CALAMACA MULA CORRAL_5MIN_20150422.csv, cols = 7\n",
      "21/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-07-03/8310_5min_20150715_subir.csv, cols = 8\n",
      "22/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-07-03/8310_5min_20150715.csv, cols = 7\n",
      "23/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-08-17/MULA CORRAL_5MIN_20150817_subir.csv, cols = 8\n",
      "24/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-08-17/MULA CORRAL_5MIN_20150817.csv, cols = 7\n",
      "25/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-09-01/MULA CORRAL_5MIN_20150901_subir.csv, cols = 8\n",
      "26/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-09-01/MULA CORRAL_5MIN_20150901.csv, cols = 7\n",
      "27/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-10-07/MULA CORRAL_5MIN_20151007_subir.csv, cols = 8\n",
      "28/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2015-10-07/MULA CORRAL_5MIN_20151007.csv, cols = 8\n",
      "29/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-01-21/8310_5min_20160126_subir.csv, cols = 8\n",
      "30/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-01-21/8310_5min_20160126.csv, cols = 7\n",
      "31/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-04-11/MULA CORRAL_5MIN_20160411_subir.csv, cols = 8\n",
      "32/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-04-11/MULA CORRAL_5MIN_20160411.csv, cols = 7\n",
      "33/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-07-11/8310_5min_20161025_subir.csv, cols = 8\n",
      "34/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-07-11/8310_5min_20161025.csv, cols = 7\n",
      "35/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-07-11/8310_1min_20161025.csv, cols = 7\n",
      "36/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-10-21/8310_5min_20161025_subir.csv, cols = 8\n",
      "37/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2016-10-21/8310_5min_20161025.csv, cols = 7\n",
      "38/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2017-01-11/MULA CORRAL_5MIN_20170111_subir.csv, cols = 8\n",
      "39/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2017-01-11/MULA CORRAL_5MIN_20170111.csv, cols = 7\n",
      "40/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-01-02/MULA CORRAL_5MIN_20180102_1_subir.csv, cols = 8\n",
      "41/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-01-02/MULA CORRAL_5MIN_20180102_2_subir.csv, cols = 8\n",
      "42/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-01-02/MULA CORRAL_5MIN_20180102_2.csv, cols = 7\n",
      "43/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-01-02/MULA CORRAL_5MIN_20180102_1.csv, cols = 7\n",
      "44/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-01-02/MULA CORRAL_5MIN_20180102.csv, cols = 7\n",
      "45/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-02-05/MULA CORRAL_5MIN_20180502_subir.csv, cols = 8\n",
      "46/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-02-05/MULA CORRAL_5MIN_20180502.csv, cols = 7\n",
      "47/60\n",
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-06-19/dia_prueba_2.csv, cols = 9\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'Date'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2645\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2646\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2647\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Date'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-381-69fd2c03c633>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'{file}/{len(files_by_date)}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_meteo_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;34mf'{path}/RAW'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfolder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfolder\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfiles_by_date\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0mdf2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreformat_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreplace_values\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreplace_values\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m         \u001b[0mmaster_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmaster_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-374-eb277ecffe40>\u001b[0m in \u001b[0;36mreformat_df\u001b[0;34m(df, replace_values)\u001b[0m\n\u001b[1;32m     40\u001b[0m                 \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Type'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnan\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m                 \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Date_Time'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_datetime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Date'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Time'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;34m'coerce'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m                 \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Date'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Time'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0minplace\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m                 \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'Date_Time'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2798\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2799\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2800\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2801\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2802\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2646\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2647\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2648\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2649\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2650\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Date'"
     ]
    }
   ],
   "source": [
    "path = '/Users/tamarahuete/Documents/Github_repos/TFM21/data'\n",
    "ziplist = glob.glob(f'{path}/RAW/*.zip')\n",
    "for folder in range(9, len(ziplist)):\n",
    "    \n",
    "    files_by_date = order_meteo_zip(path=f'{path}/RAW/', folder =folder)\n",
    "    #get_unique_variables(files_by_date,path =f'{path}/RAW/',folder = folder)\n",
    "\n",
    "    master_df =pd.DataFrame()\n",
    "    for file in range(0, len(files_by_date)):\n",
    "        print(f'{file}/{len(files_by_date)}')\n",
    "        df = read_meteo_csv(path =f'{path}/RAW',folder = folder,file = files_by_date[file])    \n",
    "        df2 = reformat_df(df=df, replace_values=replace_values)\n",
    "        master_df = master_df.append(df2)\n",
    "        \n",
    "    export_name = f'{files_by_date[0].split(\"/\")[0].replace(\" \",\"\").replace(\".\",\"\")}'\n",
    "    repeat = list(set([export_name +'.csv']) & set(os.listdir(f'{path}/PROCESSED')))\n",
    "    if len(repeat)>0:\n",
    "        master_df.to_csv(f'{path}/PROCESSED/{export_name}_{len(repeat)+1}.csv')\n",
    "    else :\n",
    "        master_df.to_csv(f'{path}/PROCESSED/{export_name}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run Meteo_utils.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(47, 9)"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file, folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success sep \";\", latin encoding\n",
      "file =16-HGPT-MT-10.- Estacion Mula Corral/2018-06-19/dia_prueba_2.csv, cols = 9\n"
     ]
    }
   ],
   "source": [
    "df = read_meteo_csv(path =f'{path}/RAW',folder = folder,file = files_by_date[file])    \n",
    "#df2 = reformat_df(df=df, replace_values=replace_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>10/21/2016,11:49:00</th>\n",
       "      <th>,</th>\n",
       "      <th>TB1minAcc</th>\n",
       "      <th>,.1</th>\n",
       "      <th>0.00</th>\n",
       "      <th>,.2</th>\n",
       "      <th>,.3</th>\n",
       "      <th>G</th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10/21/2016,11:49:00</td>\n",
       "      <td>,</td>\n",
       "      <td>RHAvg</td>\n",
       "      <td>,</td>\n",
       "      <td>101.12</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10/21/2016,11:49:00</td>\n",
       "      <td>,</td>\n",
       "      <td>ATAvg</td>\n",
       "      <td>,</td>\n",
       "      <td>-4.00</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10/21/2016,11:49:00</td>\n",
       "      <td>,</td>\n",
       "      <td>WDAvg</td>\n",
       "      <td>,</td>\n",
       "      <td>238.00</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10/21/2016,11:49:00</td>\n",
       "      <td>,</td>\n",
       "      <td>WSAvg</td>\n",
       "      <td>,</td>\n",
       "      <td>2.20</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10/21/2016,11:50:00</td>\n",
       "      <td>,</td>\n",
       "      <td>TB1minAcc</td>\n",
       "      <td>,</td>\n",
       "      <td>0.00</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3649</th>\n",
       "      <td>10/21/2016,23:59:00</td>\n",
       "      <td>,</td>\n",
       "      <td>TB1minAcc</td>\n",
       "      <td>,</td>\n",
       "      <td>0.00</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650</th>\n",
       "      <td>10/21/2016,23:59:00</td>\n",
       "      <td>,</td>\n",
       "      <td>RHAvg</td>\n",
       "      <td>,</td>\n",
       "      <td>9.89</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3651</th>\n",
       "      <td>10/21/2016,23:59:00</td>\n",
       "      <td>,</td>\n",
       "      <td>ATAvg</td>\n",
       "      <td>,</td>\n",
       "      <td>64.39</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3652</th>\n",
       "      <td>10/21/2016,23:59:00</td>\n",
       "      <td>,</td>\n",
       "      <td>WDAvg</td>\n",
       "      <td>,</td>\n",
       "      <td>188.00</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3653</th>\n",
       "      <td>10/21/2016,23:59:00</td>\n",
       "      <td>,</td>\n",
       "      <td>WSAvg</td>\n",
       "      <td>,</td>\n",
       "      <td>1.50</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>G</td>\n",
       "      <td>16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3654 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      10/21/2016,11:49:00  ,  TB1minAcc ,.1    0.00 ,.2 ,.3  G  \\\n",
       "0     10/21/2016,11:49:00  ,      RHAvg   ,  101.12   ,   ,  G   \n",
       "1     10/21/2016,11:49:00  ,      ATAvg   ,   -4.00   ,   ,  G   \n",
       "2     10/21/2016,11:49:00  ,      WDAvg   ,  238.00   ,   ,  G   \n",
       "3     10/21/2016,11:49:00  ,      WSAvg   ,    2.20   ,   ,  G   \n",
       "4     10/21/2016,11:50:00  ,  TB1minAcc   ,    0.00   ,   ,  G   \n",
       "...                   ... ..        ...  ..     ...  ..  .. ..   \n",
       "3649  10/21/2016,23:59:00  ,  TB1minAcc   ,    0.00   ,   ,  G   \n",
       "3650  10/21/2016,23:59:00  ,      RHAvg   ,    9.89   ,   ,  G   \n",
       "3651  10/21/2016,23:59:00  ,      ATAvg   ,   64.39   ,   ,  G   \n",
       "3652  10/21/2016,23:59:00  ,      WDAvg   ,  188.00   ,   ,  G   \n",
       "3653  10/21/2016,23:59:00  ,      WSAvg   ,    1.50   ,   ,  G   \n",
       "\n",
       "                                              file_name  \n",
       "0     16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "1     16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "2     16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "3     16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "4     16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "...                                                 ...  \n",
       "3649  16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "3650  16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "3651  16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "3652  16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "3653  16-HGPT-MT-10.- Estacion Mula Corral/2018-06-1...  \n",
       "\n",
       "[3654 rows x 9 columns]"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = reformat_df(df=df, replace_values=replace_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder =8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [],
   "source": [
    "ziplist = glob.glob(f'{path}/RAW/*.zip')\n",
    "zf = zipfile.ZipFile(f'{ziplist[folder]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [],
   "source": [
    "if len(df.columns) == 1:\n",
    "        df = df[df.columns[0]].str.split(',',expand=True)\n",
    "        if len(df.columns) == 1:\n",
    "            df = df[df.columns[0]].str.split('.',expand=True)\n",
    "            df[3] =df[3].astype(str).str.cat(df[4].astype(str),sep=\".\")\n",
    "            df.drop(columns=4, inplace=True)\n",
    "        df = df.iloc[1:]   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Variable</th>\n",
       "      <th>Value</th>\n",
       "      <th>Type</th>\n",
       "      <th>Freq</th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>06/21/2013</td>\n",
       "      <td>10:45:00</td>\n",
       "      <td>TB5minAcc</td>\n",
       "      <td>0.00</td>\n",
       "      <td>G</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>06/21/2013</td>\n",
       "      <td>10:45:00</td>\n",
       "      <td>AT5minAvg</td>\n",
       "      <td>14.55</td>\n",
       "      <td>B</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>06/21/2013</td>\n",
       "      <td>10:45:00</td>\n",
       "      <td>AT5minMin</td>\n",
       "      <td>14.41</td>\n",
       "      <td>G</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>06/21/2013</td>\n",
       "      <td>10:45:00</td>\n",
       "      <td>AT5minMax</td>\n",
       "      <td>14.69</td>\n",
       "      <td>G</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>06/21/2013</td>\n",
       "      <td>10:45:00</td>\n",
       "      <td>RH5minAvg</td>\n",
       "      <td>71.89</td>\n",
       "      <td>B</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153306</th>\n",
       "      <td>08/01/2013</td>\n",
       "      <td>09:25:00</td>\n",
       "      <td>WS5minAvg</td>\n",
       "      <td>1.10</td>\n",
       "      <td>G</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153307</th>\n",
       "      <td>08/01/2013</td>\n",
       "      <td>09:25:00</td>\n",
       "      <td>WS5minMin</td>\n",
       "      <td>0.95</td>\n",
       "      <td>G</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153308</th>\n",
       "      <td>08/01/2013</td>\n",
       "      <td>09:25:00</td>\n",
       "      <td>WS5minMax</td>\n",
       "      <td>1.30</td>\n",
       "      <td>G</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153309</th>\n",
       "      <td>08/01/2013</td>\n",
       "      <td>09:25:00</td>\n",
       "      <td>WD5minMax</td>\n",
       "      <td>-65.03</td>\n",
       "      <td>G</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153310</th>\n",
       "      <td>\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None.None</td>\n",
       "      <td>None</td>\n",
       "      <td>5min</td>\n",
       "      <td>09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>153310 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     Date      Time  \\\n",
       "1                                              06/21/2013  10:45:00   \n",
       "2                                              06/21/2013  10:45:00   \n",
       "3                                              06/21/2013  10:45:00   \n",
       "4                                              06/21/2013  10:45:00   \n",
       "5                                              06/21/2013  10:45:00   \n",
       "...                                                   ...       ...   \n",
       "153306                                         08/01/2013  09:25:00   \n",
       "153307                                         08/01/2013  09:25:00   \n",
       "153308                                         08/01/2013  09:25:00   \n",
       "153309                                         08/01/2013  09:25:00   \n",
       "153310  \u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a\u001a...      None   \n",
       "\n",
       "         Variable      Value  Type  Freq  \\\n",
       "1       TB5minAcc       0.00     G  5min   \n",
       "2       AT5minAvg      14.55     B  5min   \n",
       "3       AT5minMin      14.41     G  5min   \n",
       "4       AT5minMax      14.69     G  5min   \n",
       "5       RH5minAvg      71.89     B  5min   \n",
       "...           ...        ...   ...   ...   \n",
       "153306  WS5minAvg       1.10     G  5min   \n",
       "153307  WS5minMin       0.95     G  5min   \n",
       "153308  WS5minMax       1.30     G  5min   \n",
       "153309  WD5minMax     -65.03     G  5min   \n",
       "153310       None  None.None  None  5min   \n",
       "\n",
       "                                                file_name  \n",
       "1       09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "2       09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "3       09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "4       09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "5       09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "...                                                   ...  \n",
       "153306  09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "153307  09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "153308  09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "153309  09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "153310  09-HGPT-MT-02.- Estacion Colegio A. J. Sucre/2...  \n",
       "\n",
       "[153310 rows x 7 columns]"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}