P5_DataFrame.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9be12b7-522a-4da7-bcab-35d381b6aae0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8ac2604b-7783-45f8-856b-f152198654c7",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 1.1 : Importation des packages & fonctions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "66a5f6e0-75b0-4f76-bce6-5b58885d8521",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keyboard in c:\\users\\evari\\anaconda3\\lib\\site-packages (0.13.5)\n",
      "Requirement already satisfied: bs4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (0.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from bs4) (4.9.3)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from beautifulsoup4->bs4) (2.2.1)\n",
      "Requirement already satisfied: autocorrect in c:\\users\\evari\\anaconda3\\lib\\site-packages (2.5.0)\n",
      "Requirement already satisfied: yellowbrick in c:\\users\\evari\\appdata\\roaming\\python\\python38\\site-packages (1.3.post1)\n",
      "Requirement already satisfied: scikit-learn>=0.20 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (0.24.1)\n",
      "Requirement already satisfied: scipy>=1.0.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (1.6.2)\n",
      "Requirement already satisfied: numpy<1.20,>=1.16.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (1.19.5)\n",
      "Requirement already satisfied: cycler>=0.10.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (0.10.0)\n",
      "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.2 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (3.3.4)\n",
      "Requirement already satisfied: six in c:\\users\\evari\\anaconda3\\lib\\site-packages (from cycler>=0.10.0->yellowbrick) (1.15.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (2.8.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (1.3.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (8.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from scikit-learn>=0.20->yellowbrick) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from scikit-learn>=0.20->yellowbrick) (1.0.1)\n"
     ]
    }
   ],
   "source": [
    "# Importation des packages souhaités\n",
    "%run P5_Packages.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e57f9689-5e89-4aa8-94ab-cf5fb8059f66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keyboard in c:\\users\\evari\\anaconda3\\lib\\site-packages (0.13.5)\n",
      "Requirement already satisfied: bs4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (0.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from bs4) (4.9.3)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from beautifulsoup4->bs4) (2.2.1)\n",
      "Requirement already satisfied: autocorrect in c:\\users\\evari\\anaconda3\\lib\\site-packages (2.5.0)\n",
      "Requirement already satisfied: yellowbrick in c:\\users\\evari\\appdata\\roaming\\python\\python38\\site-packages (1.3.post1)\n",
      "Requirement already satisfied: scipy>=1.0.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (1.6.2)\n",
      "Requirement already satisfied: scikit-learn>=0.20 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (0.24.1)\n",
      "Requirement already satisfied: numpy<1.20,>=1.16.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (1.19.5)\n",
      "Requirement already satisfied: cycler>=0.10.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (0.10.0)\n",
      "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.2 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from yellowbrick) (3.3.4)\n",
      "Requirement already satisfied: six in c:\\users\\evari\\anaconda3\\lib\\site-packages (from cycler>=0.10.0->yellowbrick) (1.15.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (2.8.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (1.3.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (8.2.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (2.4.7)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from scikit-learn>=0.20->yellowbrick) (1.0.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from scikit-learn>=0.20->yellowbrick) (2.1.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\evari\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "# Importation du notebook contenant les fonctions\n",
    "%run P5_Functions.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9226cb51-27eb-4880-8dbd-a907ce52cbc5",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 1.2 : Ouverture des fichiers csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "38af144e-494b-4dab-9408-a318c1e83dbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e1595a37-4011-452c-8c20-4aa1ac72cd6f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "QueryResults\\QueryResults_01.csv\n",
      "QueryResults\\QueryResults_02.csv\n"
     ]
    }
   ],
   "source": [
    "# Affichage des fichiers présents\n",
    "list_of_files = []\n",
    "\n",
    "for file in glob.glob('QueryResults\\*.csv'):\n",
    "    list_of_files.append(file)\n",
    "    print(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1cc5fd90-2669-4439-8fb9-c782db61ec4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ln = len(list_of_files)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da11a5ae-f54c-4556-9574-8ae422582547",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 1.3 : Construction des dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8d4f2cf8-bb15-4304-bec5-b3bfe0995ad9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'data_1', 1: 'data_2'}\n"
     ]
    }
   ],
   "source": [
    "data_names = {i:'data_{}'.format(i+1) for i in range(0,ln)}\n",
    "\n",
    "print(data_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "695842a5-af0f-4ad5-888a-211c03b12e10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data_2'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_names[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5294bf39-336c-4d3b-a7d2-70fccddd9d21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'df_1', 1: 'df_2'}\n"
     ]
    }
   ],
   "source": [
    "df_names = {i:'df_{}'.format(i+1) for i in range(0,ln)}\n",
    "\n",
    "print(df_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fd1fc174-8be5-43d3-bfe5-b28a902f1b7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'df_1'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7a992ddc-0050-4753-b01d-65048a081ba9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# values = df_names.values()\n",
    "# values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "676f9fef-e6ea-41f3-b868-5ead5d8ef562",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Convert all values in dictionary to a list\n",
    "# list_of_names = [value for key, value in df_names.items()]\n",
    "# print(list_of_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "71d7818a-8f29-43ce-8bd3-038a4929a6c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list_of_names[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b04b0321-c40c-442f-908e-2dacce9eb8c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_1\n",
      "QueryResults\\QueryResults_01.csv\n",
      "(50000, 11)\n",
      "data_2\n",
      "QueryResults\\QueryResults_02.csv\n",
      "(50000, 11)\n"
     ]
    }
   ],
   "source": [
    "# Creation des dataframes de base\n",
    "# On importe les csv pas à pas dans un dataframe\n",
    "\n",
    "data = {}\n",
    "for i in range(0,ln):\n",
    "    name = data_names[i]\n",
    "    importation = list_of_files[i]\n",
    "    data[name] = pd.read_csv(importation, sep=',')\n",
    "\n",
    "    print(name)\n",
    "    print(list_of_files[i])\n",
    "    print(data[name].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c4ced18-00d3-4da1-9a27-a669ab5d62bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_names = ['data_1',\n",
    "#               'data_2',\n",
    "#               'data_3',\n",
    "#               'data_4',\n",
    "#               'data_5',\n",
    "#               'data_6',\n",
    "#               'data_7',\n",
    "#               'data_8',\n",
    "#               'data_9',\n",
    "#               'data_10',\n",
    "#               'data_11',\n",
    "#               'data_12',\n",
    "#               'data_13',\n",
    "#               'data_14',\n",
    "#               'data_15',\n",
    "#               'data_16',\n",
    "#               'data_17',\n",
    "#               'data_18',\n",
    "#               'data_19',\n",
    "#               'data_20']\n",
    "\n",
    "# df_names = ['df_1',\n",
    "#             'df_2',\n",
    "#             'df_3',\n",
    "#             'df_4',\n",
    "#             'df_5',\n",
    "#             'df_6',\n",
    "#             'df_7',\n",
    "#             'df_8',\n",
    "#             'df_9',\n",
    "#             'df_10',\n",
    "#             'df_11',\n",
    "#             'df_12',\n",
    "#             'df_13',\n",
    "#             'df_14',\n",
    "#             'df_15',\n",
    "#             'df_16',\n",
    "#             'df_17',\n",
    "#             'df_18',\n",
    "#             'df_19',\n",
    "#             'df_20']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47571ada-5cb1-46f6-b59d-0a0118683060",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bed1612-6d73-4bc7-a7ef-c2646febbdcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Creation des dataframes df en copie\n",
    "\n",
    "# df = {}\n",
    "# i = 0\n",
    "# for name in df_names:\n",
    "#     name_i = data_names[i]\n",
    "#     df[name] = data[name_i].copy()\n",
    "\n",
    "#     print(name)\n",
    "#     print(df[name].shape)\n",
    "    \n",
    "#     # incrémentation sur i\n",
    "#     i = i + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b4c5846-dfd3-4452-86dc-a9c146f0c667",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Commande pour acceder au dataframe\n",
    "data['data_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b9a106b-f91f-472a-baf0-f37cea638b6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creation des dataframes df en copie\n",
    "\n",
    "df = {}\n",
    "for i in range(0,ln):\n",
    "    \n",
    "    name = df_names[i]\n",
    "    name_2 = data_names[i]\n",
    "    df[name] = data[name_2].copy()\n",
    "\n",
    "    print(name)\n",
    "    print(df[name].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e54a941-e0ae-4f33-be75-39c80ee5cdee",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['df_1'].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9f78046-4fd8-4266-ada3-5b60cad96dfc",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 1.4 : Concaténation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff49cfcc-4969-4e84-976f-99a78a106a28",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataframe total : concaténer\n",
    "\n",
    "# On initialise le dataframe avec le 1er element\n",
    "df_raw = df['df_1'].copy()\n",
    "\n",
    "for i in range(1, len(df_names)):\n",
    "    \n",
    "    # Le df à récupérer\n",
    "    name_i = df_names[i]\n",
    "    \n",
    "    # Concaténation à chaque iteration\n",
    "    df_raw = pd.concat([df_raw, df[name_i]], ignore_index=True)\n",
    "    \n",
    "    # Affichage\n",
    "    print('df_raw', '+', name_i)\n",
    "    print(df_raw .shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3da87e1-7b57-417b-b9bf-1d7940c9fced",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9037f704-e016-4b31-9371-0b7e8dfe57ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef07fb73-f942-4277-bc3a-7d995bd1f479",
   "metadata": {},
   "source": [
    "# Remove duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e9486a8-0500-48b4-9157-ddc23ecd9cb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw = df_raw.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d928e8b9-9cf8-4b01-aa9a-28ffd1182e91",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "660ae566-d69d-4465-b3d3-097bca2a9e1b",
   "metadata": {},
   "source": [
    "# Save Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4614a8ad-76df-45fa-bb03-02ae4d0d90d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Année 2020 et 2021\n",
    "df_raw.to_csv(r'Fichiers\\df_raw.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed26e12b-7fc7-49c6-bb4a-3a3d45e2ddcd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}