P5_Nettoyage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "df92f39a-cd12-4a4a-ba13-8c27a9928ae9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import warnings\n",
    "\n",
    "# warnings.filterwarnings(action='always')\n",
    "# warnings.filterwarnings(\"ignore\",category=DeprecationWarning)\n",
    "# warnings.filterwarnings(\"ignore\",category=ResourceWarning)\n",
    "\n",
    "import sys\n",
    "import warnings\n",
    "\n",
    "if not sys.warnoptions:\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    \n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f99100d-4a89-4e81-9733-0a928b58fc69",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 1.1 : Importation des packages & fonctions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a4dcb0b5-9634-4c1a-a3f8-b86eef5e37b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keyboard in c:\\users\\evari\\anaconda3\\lib\\site-packages (0.13.5)\n",
      "Requirement already satisfied: bs4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (0.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from bs4) (4.9.3)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from beautifulsoup4->bs4) (2.2.1)\n",
      "Requirement already satisfied: autocorrect in c:\\users\\evari\\anaconda3\\lib\\site-packages (2.5.0)\n",
      "Requirement already satisfied: wordcloud in c:\\users\\evari\\anaconda3\\lib\\site-packages (1.8.1)\n",
      "Requirement already satisfied: pillow in c:\\users\\evari\\anaconda3\\lib\\site-packages (from wordcloud) (8.2.0)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\evari\\anaconda3\\lib\\site-packages (from wordcloud) (3.3.4)\n",
      "Requirement already satisfied: numpy>=1.6.1 in c:\\users\\evari\\appdata\\roaming\\python\\python38\\site-packages (from wordcloud) (1.19.5)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib->wordcloud) (1.3.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib->wordcloud) (2.4.7)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib->wordcloud) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib->wordcloud) (2.8.1)\n",
      "Requirement already satisfied: six in c:\\users\\evari\\anaconda3\\lib\\site-packages (from cycler>=0.10->matplotlib->wordcloud) (1.15.0)\n",
      "Requirement already satisfied: pyLDAvis in c:\\users\\evari\\anaconda3\\lib\\site-packages (3.2.1)\n",
      "Requirement already satisfied: wheel>=0.23.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (0.36.2)\n",
      "Requirement already satisfied: jinja2>=2.7.2 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (2.11.3)\n",
      "Requirement already satisfied: numpy>=1.9.2 in c:\\users\\evari\\appdata\\roaming\\python\\python38\\site-packages (from pyLDAvis) (1.19.5)\n",
      "Requirement already satisfied: joblib>=0.8.4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (1.0.1)\n",
      "Requirement already satisfied: scipy>=0.18.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (1.6.2)\n",
      "Requirement already satisfied: numexpr in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (2.7.3)\n",
      "Requirement already satisfied: future in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (0.18.2)\n",
      "Requirement already satisfied: funcy in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (1.16)\n",
      "Requirement already satisfied: pandas>=0.17.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (1.2.4)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from jinja2>=2.7.2->pyLDAvis) (1.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pandas>=0.17.0->pyLDAvis) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.3 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pandas>=0.17.0->pyLDAvis) (2021.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from python-dateutil>=2.7.3->pandas>=0.17.0->pyLDAvis) (1.15.0)\n"
     ]
    }
   ],
   "source": [
    "# Importation des packages souhaités\n",
    "%run P5_Packages.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "87b20efc-d4a3-400f-9dbb-e4864251254e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keyboard in c:\\users\\evari\\anaconda3\\lib\\site-packages (0.13.5)\n",
      "Requirement already satisfied: bs4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (0.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from bs4) (4.9.3)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from beautifulsoup4->bs4) (2.2.1)\n",
      "Requirement already satisfied: autocorrect in c:\\users\\evari\\anaconda3\\lib\\site-packages (2.5.0)\n",
      "Requirement already satisfied: wordcloud in c:\\users\\evari\\anaconda3\\lib\\site-packages (1.8.1)\n",
      "Requirement already satisfied: pillow in c:\\users\\evari\\anaconda3\\lib\\site-packages (from wordcloud) (8.2.0)\n",
      "Requirement already satisfied: numpy>=1.6.1 in c:\\users\\evari\\appdata\\roaming\\python\\python38\\site-packages (from wordcloud) (1.19.5)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\evari\\anaconda3\\lib\\site-packages (from wordcloud) (3.3.4)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib->wordcloud) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib->wordcloud) (2.8.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib->wordcloud) (1.3.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from matplotlib->wordcloud) (2.4.7)\n",
      "Requirement already satisfied: six in c:\\users\\evari\\anaconda3\\lib\\site-packages (from cycler>=0.10->matplotlib->wordcloud) (1.15.0)\n",
      "Requirement already satisfied: pyLDAvis in c:\\users\\evari\\anaconda3\\lib\\site-packages (3.2.1)\n",
      "Requirement already satisfied: wheel>=0.23.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (0.36.2)\n",
      "Requirement already satisfied: numexpr in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (2.7.3)\n",
      "Requirement already satisfied: jinja2>=2.7.2 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (2.11.3)\n",
      "Requirement already satisfied: future in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (0.18.2)\n",
      "Requirement already satisfied: pandas>=0.17.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (1.2.4)\n",
      "Requirement already satisfied: scipy>=0.18.0 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (1.6.2)\n",
      "Requirement already satisfied: numpy>=1.9.2 in c:\\users\\evari\\appdata\\roaming\\python\\python38\\site-packages (from pyLDAvis) (1.19.5)\n",
      "Requirement already satisfied: joblib>=0.8.4 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (1.0.1)\n",
      "Requirement already satisfied: funcy in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pyLDAvis) (1.16)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from jinja2>=2.7.2->pyLDAvis) (1.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pandas>=0.17.0->pyLDAvis) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.3 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from pandas>=0.17.0->pyLDAvis) (2021.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\evari\\anaconda3\\lib\\site-packages (from python-dateutil>=2.7.3->pandas>=0.17.0->pyLDAvis) (1.15.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\evari\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "# Importation du notebook contenant les fonctions\n",
    "%run P5_Functions.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83e9ab8d-e297-46f3-97bc-5f73cc710faa",
   "metadata": {},
   "source": [
    "# 1.2 : Importation des fichiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e8ee8c80-c0cc-4c79-86ec-97311d17026b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "b66357c2-26a3-469d-b1aa-568e7947165b",
   "metadata": {},
   "outputs": [],
   "source": [
    "importation = 'Fichiers\\df_raw.csv'\n",
    "df_questions = pd.read_csv(importation, sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "a18f44cb-d200-4323-8f80-8fabfabafa19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>doc</th>\n",
       "      <th>title</th>\n",
       "      <th>tags</th>\n",
       "      <th>creation_date</th>\n",
       "      <th>score</th>\n",
       "      <th>views</th>\n",
       "      <th>answers</th>\n",
       "      <th>comments</th>\n",
       "      <th>favorites</th>\n",
       "      <th>last_activity_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>9056957</td>\n",
       "      <td>&lt;p&gt;I noticed that in Python, people initialize...</td>\n",
       "      <td>correct way to define class variables in Python</td>\n",
       "      <td>&lt;python&gt;&lt;class&gt;&lt;attributes&gt;</td>\n",
       "      <td>2012-01-29 21:26:39</td>\n",
       "      <td>296</td>\n",
       "      <td>372888</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>105</td>\n",
       "      <td>2021-01-25 13:17:29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>52013133</td>\n",
       "      <td>&lt;p&gt;I'm using Webpack 4 and I'm creating the co...</td>\n",
       "      <td>Entrypoint undefined = index.html using HtmlWe...</td>\n",
       "      <td>&lt;node.js&gt;&lt;reactjs&gt;&lt;webpack&gt;&lt;html-webpack-plugin&gt;</td>\n",
       "      <td>2018-08-25 00:32:17</td>\n",
       "      <td>31</td>\n",
       "      <td>33607</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2021-07-18 06:19:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>261963</td>\n",
       "      <td>&lt;p&gt;I just noticed that you can not use standar...</td>\n",
       "      <td>How can I iterate over an enum?</td>\n",
       "      <td>&lt;c++&gt;&lt;enums&gt;</td>\n",
       "      <td>2008-11-04 13:55:29</td>\n",
       "      <td>354</td>\n",
       "      <td>374899</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>67</td>\n",
       "      <td>2021-05-23 11:33:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>60684868</td>\n",
       "      <td>&lt;p&gt;I'm currently trying to use &lt;a href=\"https:...</td>\n",
       "      <td>Docker Image using compiled TensorFlow library...</td>\n",
       "      <td>&lt;docker&gt;&lt;tensorflow&gt;</td>\n",
       "      <td>2020-03-14 16:25:39</td>\n",
       "      <td>1</td>\n",
       "      <td>225</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-01-04 03:48:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>35136616</td>\n",
       "      <td>&lt;p&gt;I can't figure out what to do in case of we...</td>\n",
       "      <td>handling WebGL loss in three js</td>\n",
       "      <td>&lt;javascript&gt;&lt;three.js&gt;&lt;webgl&gt;&lt;electron&gt;</td>\n",
       "      <td>2016-02-01 17:26:01</td>\n",
       "      <td>6</td>\n",
       "      <td>412</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-03-18 17:02:54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0        id                                                doc  \\\n",
       "0           0   9056957  <p>I noticed that in Python, people initialize...   \n",
       "1           1  52013133  <p>I'm using Webpack 4 and I'm creating the co...   \n",
       "2           2    261963  <p>I just noticed that you can not use standar...   \n",
       "3           3  60684868  <p>I'm currently trying to use <a href=\"https:...   \n",
       "4           4  35136616  <p>I can't figure out what to do in case of we...   \n",
       "\n",
       "                                               title  \\\n",
       "0    correct way to define class variables in Python   \n",
       "1  Entrypoint undefined = index.html using HtmlWe...   \n",
       "2                    How can I iterate over an enum?   \n",
       "3  Docker Image using compiled TensorFlow library...   \n",
       "4                    handling WebGL loss in three js   \n",
       "\n",
       "                                               tags        creation_date  \\\n",
       "0                       <python><class><attributes>  2012-01-29 21:26:39   \n",
       "1  <node.js><reactjs><webpack><html-webpack-plugin>  2018-08-25 00:32:17   \n",
       "2                                      <c++><enums>  2008-11-04 13:55:29   \n",
       "3                              <docker><tensorflow>  2020-03-14 16:25:39   \n",
       "4           <javascript><three.js><webgl><electron>  2016-02-01 17:26:01   \n",
       "\n",
       "   score   views  answers  comments  favorites   last_activity_date  \n",
       "0    296  372888        2         1        105  2021-01-25 13:17:29  \n",
       "1     31   33607        5         2          7  2021-07-18 06:19:28  \n",
       "2    354  374899       24         5         67  2021-05-23 11:33:10  \n",
       "3      1     225        1         5          1  2021-01-04 03:48:09  \n",
       "4      6     412        1         0          1  2021-03-18 17:02:54  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_questions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e80d4ab4-543a-4d71-b132-2e55c77d7ce4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>doc</th>\n",
       "      <th>title</th>\n",
       "      <th>tags</th>\n",
       "      <th>creation_date</th>\n",
       "      <th>score</th>\n",
       "      <th>views</th>\n",
       "      <th>answers</th>\n",
       "      <th>comments</th>\n",
       "      <th>favorites</th>\n",
       "      <th>last_activity_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>9056957</td>\n",
       "      <td>&lt;p&gt;I noticed that in Python, people initialize...</td>\n",
       "      <td>correct way to define class variables in Python</td>\n",
       "      <td>&lt;python&gt;&lt;class&gt;&lt;attributes&gt;</td>\n",
       "      <td>2012-01-29 21:26:39</td>\n",
       "      <td>296</td>\n",
       "      <td>372888</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>105</td>\n",
       "      <td>2021-01-25 13:17:29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>52013133</td>\n",
       "      <td>&lt;p&gt;I'm using Webpack 4 and I'm creating the co...</td>\n",
       "      <td>Entrypoint undefined = index.html using HtmlWe...</td>\n",
       "      <td>&lt;node.js&gt;&lt;reactjs&gt;&lt;webpack&gt;&lt;html-webpack-plugin&gt;</td>\n",
       "      <td>2018-08-25 00:32:17</td>\n",
       "      <td>31</td>\n",
       "      <td>33607</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2021-07-18 06:19:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>261963</td>\n",
       "      <td>&lt;p&gt;I just noticed that you can not use standar...</td>\n",
       "      <td>How can I iterate over an enum?</td>\n",
       "      <td>&lt;c++&gt;&lt;enums&gt;</td>\n",
       "      <td>2008-11-04 13:55:29</td>\n",
       "      <td>354</td>\n",
       "      <td>374899</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>67</td>\n",
       "      <td>2021-05-23 11:33:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>60684868</td>\n",
       "      <td>&lt;p&gt;I'm currently trying to use &lt;a href=\"https:...</td>\n",
       "      <td>Docker Image using compiled TensorFlow library...</td>\n",
       "      <td>&lt;docker&gt;&lt;tensorflow&gt;</td>\n",
       "      <td>2020-03-14 16:25:39</td>\n",
       "      <td>1</td>\n",
       "      <td>225</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-01-04 03:48:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>35136616</td>\n",
       "      <td>&lt;p&gt;I can't figure out what to do in case of we...</td>\n",
       "      <td>handling WebGL loss in three js</td>\n",
       "      <td>&lt;javascript&gt;&lt;three.js&gt;&lt;webgl&gt;&lt;electron&gt;</td>\n",
       "      <td>2016-02-01 17:26:01</td>\n",
       "      <td>6</td>\n",
       "      <td>412</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-03-18 17:02:54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0        id                                                doc  \\\n",
       "0           0   9056957  <p>I noticed that in Python, people initialize...   \n",
       "1           1  52013133  <p>I'm using Webpack 4 and I'm creating the co...   \n",
       "2           2    261963  <p>I just noticed that you can not use standar...   \n",
       "3           3  60684868  <p>I'm currently trying to use <a href=\"https:...   \n",
       "4           4  35136616  <p>I can't figure out what to do in case of we...   \n",
       "\n",
       "                                               title  \\\n",
       "0    correct way to define class variables in Python   \n",
       "1  Entrypoint undefined = index.html using HtmlWe...   \n",
       "2                    How can I iterate over an enum?   \n",
       "3  Docker Image using compiled TensorFlow library...   \n",
       "4                    handling WebGL loss in three js   \n",
       "\n",
       "                                               tags        creation_date  \\\n",
       "0                       <python><class><attributes>  2012-01-29 21:26:39   \n",
       "1  <node.js><reactjs><webpack><html-webpack-plugin>  2018-08-25 00:32:17   \n",
       "2                                      <c++><enums>  2008-11-04 13:55:29   \n",
       "3                              <docker><tensorflow>  2020-03-14 16:25:39   \n",
       "4           <javascript><three.js><webgl><electron>  2016-02-01 17:26:01   \n",
       "\n",
       "   score   views  answers  comments  favorites   last_activity_date  \n",
       "0    296  372888        2         1        105  2021-01-25 13:17:29  \n",
       "1     31   33607        5         2          7  2021-07-18 06:19:28  \n",
       "2    354  374899       24         5         67  2021-05-23 11:33:10  \n",
       "3      1     225        1         5          1  2021-01-04 03:48:09  \n",
       "4      6     412        1         0          1  2021-03-18 17:02:54  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_questions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "67b74aa3-676f-469b-aaf4-86636154f566",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Attention à l'encodage\n",
    "df_questions[['title', 'doc']] = df_questions[['title','doc']].applymap(lambda x: str(x).encode(\"utf-8\", errors='surrogatepass').decode(\"ISO-8859-1\", errors='surrogatepass'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "8085675e-88d1-42ac-b108-2bfae7e775f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0            0.0\n",
       "id                    0.0\n",
       "doc                   0.0\n",
       "title                 0.0\n",
       "tags                  0.0\n",
       "creation_date         0.0\n",
       "score                 0.0\n",
       "views                 0.0\n",
       "answers               0.0\n",
       "comments              0.0\n",
       "favorites             0.0\n",
       "last_activity_date    0.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_questions[df_questions['score']<=0].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5c0c51fd-f659-435c-a0d7-0504ae07c769",
   "metadata": {},
   "outputs": [],
   "source": [
    "spell = Speller()\n",
    "token = ToktokTokenizer()\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "stemmer = PorterStemmer()\n",
    "charac = '!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~0123456789'\n",
    "stop_words = set(stopwords.words(\"english\"))\n",
    "\n",
    "# List of Adjective's tag from nltk package\n",
    "adjective_tag_list = set(['JJ','JJR', 'JJS', 'RBR', 'RBS']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "a7a6dca5-aaa4-4eda-841b-d0c21037ff34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 99986 entries, 0 to 99985\n",
      "Data columns (total 12 columns):\n",
      " #   Column              Non-Null Count  Dtype \n",
      "---  ------              --------------  ----- \n",
      " 0   Unnamed: 0          99986 non-null  int64 \n",
      " 1   id                  99986 non-null  int64 \n",
      " 2   doc                 99986 non-null  object\n",
      " 3   title               99986 non-null  object\n",
      " 4   tags                99986 non-null  object\n",
      " 5   creation_date       99986 non-null  object\n",
      " 6   score               99986 non-null  int64 \n",
      " 7   views               99986 non-null  int64 \n",
      " 8   answers             99986 non-null  int64 \n",
      " 9   comments            99986 non-null  int64 \n",
      " 10  favorites           99986 non-null  int64 \n",
      " 11  last_activity_date  99986 non-null  object\n",
      "dtypes: int64(7), object(5)\n",
      "memory usage: 9.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df_questions.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "bbb0fac4-c6df-47cb-9edb-b5dd56660f36",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Temps dépensé pour le tout\n",
    "start_all = timeit.default_timer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "ff662349-b2e6-491a-be78-40df0aa0514f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose doc num to print\n",
    "N_doc = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90c1b275-4a84-4f31-bc49-fdf8b39304ce",
   "metadata": {},
   "source": [
    "# 1.3 : Noise removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "19f965b6-25ea-4001-aaa4-cb50ee013065",
   "metadata": {},
   "outputs": [],
   "source": [
    "start_all = timeit.default_timer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "d37a25e5-03f9-419d-82ca-86c654f9dd40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p>I noticed that in Python, people initialize their class attributes in two different ways.</p>\n",
      "<p>The first way is like this:</p>\n",
      "<pre><code>class MyClass:\n",
      "  __element1 = 123\n",
      "  __element2 = &quot;this is Africa&quot;\n",
      "\n",
      "  def __init__(self):\n",
      "    #pass or something else\n",
      "</code></pre>\n",
      "<p>The other style looks like:</p>\n",
      "<pre><code>class MyClass:\n",
      "  def __init__(self):\n",
      "    self.__element1 = 123\n",
      "    self.__element2 = &quot;this is Africa&quot;\n",
      "</code></pre>\n",
      "<p>Which is the correct way to initialize class attributes?</p>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Exemple de donnée dans colonne doc\n",
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "91313ca5-f25c-4a18-a1e0-2eb79d5819a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Parse question and title then return only the text\n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text())\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5ae30066-ba75-4e77-99bf-b72669e70c1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The simple HTML below displays differently in Firefox and WebKit-based browsers (I checked in Safari, Chrome and iPhone).\n",
      "In Firefox both border and text have the same color (#880000), but in Safari the text gets a bit lighter (as if it had some transparency applied to it).\n",
      "Can I somehow fix this (remove this transparency in Safari)?\n",
      "\n",
      "\n",
      "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n",
      "    <head>\n",
      "        <title></title>\n",
      "        <style type=\"text/css\">\n",
      "        input:disabled{\n",
      "            border:solid 1px #880000;\n",
      "            background-color:#ffffff;\n",
      "            color:#880000;\n",
      "        }\n",
      "        </style>\n",
      "    </head>\n",
      "    <body>\n",
      "        <form action=\"\">\n",
      "            <input type=\"text\" value=\"disabled input box\" disabled=\"disabled\"/>\n",
      "        </form>\n",
      "    </body>\n",
      "</html>\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "# Exemple de donnée dans colonne doc\n",
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cc9dcb32-a6fe-4272-bdf9-ae06b85629ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n",
      "<>:5: DeprecationWarning: invalid escape sequence \\s\n",
      "<>:5: DeprecationWarning: invalid escape sequence \\s\n",
      "<ipython-input-22-3df81b83692b>:5: DeprecationWarning: invalid escape sequence \\s\n",
      "  text = re.sub('\\s+', ' ', text) # match all one or more whitespace then replace them by a single whitespace\n"
     ]
    }
   ],
   "source": [
    "def clean_text(text):\n",
    "    text = re.sub(r\"\\'\", \"'\", text) # match all literal apostrophe pattern then replace them by a single whitespace\n",
    "    text = re.sub(r\"\\n\", \" \", text) # match all literal Line Feed (New line) pattern then replace them by a single whitespace\n",
    "    text = re.sub(r\"\\xa0\", \" \", text) # match all literal non-breakable space pattern then replace them by a single whitespace\n",
    "    text = re.sub('\\s+', ' ', text) # match all one or more whitespace then replace them by a single whitespace\n",
    "    text = text.strip(' ')\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0435556a-314d-4a2d-a033-d5a86c946b87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 8.02 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: clean_text(x)) \n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2bd811dc-0c35-4b74-95ce-62843524b123",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The simple HTML below displays differently in Firefox and WebKit-based browsers (I checked in Safari, Chrome and iPhone). In Firefox both border and text have the same color (#880000), but in Safari the text gets a bit lighter (as if it had some transparency applied to it). Can I somehow fix this (remove this transparency in Safari)? <html xmlns=\"http://www.w3.org/1999/xhtml\"> <head> <title></title> <style type=\"text/css\"> input:disabled{ border:solid 1px #880000; background-color:#ffffff; color:#880000; } </style> </head> <body> <form action=\"\"> <input type=\"text\" value=\"disabled input box\" disabled=\"disabled\"/> </form> </body> </html>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a1b07e26-5ca7-4de8-888c-d2aca92865f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "def expand_contractions(text):\n",
    "    \"\"\"expand shortened words, e.g. 'don't' to 'do not'\"\"\"\n",
    "    text = contractions.fix(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "dabe8435-837b-46e1-a000-1f4c608ba220",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "def expand_contractions(text, contraction_mapping=CONTRACTION_MAP):\n",
    "    \n",
    "    contractions_pattern = re.compile('({})'.format('|'.join(contraction_mapping.keys())), \n",
    "                                      flags=re.IGNORECASE|re.DOTALL)\n",
    "    def expand_match(contraction):\n",
    "        match = contraction.group(0)\n",
    "        first_char = match[0]\n",
    "        expanded_contraction = contraction_mapping.get(match)\\\n",
    "                                if contraction_mapping.get(match)\\\n",
    "                                else contraction_mapping.get(match.lower())                       \n",
    "        expanded_contraction = first_char+expanded_contraction[1:]\n",
    "        return expanded_contraction\n",
    "        \n",
    "    expanded_text = contractions_pattern.sub(expand_match, text)\n",
    "    expanded_text = re.sub(\"'\", \"\", expanded_text)\n",
    "    return expanded_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2e8d63d1-e1bb-4ad7-8b5a-4adc1616b9f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: expand_contractions(x)) \n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: expand_contractions(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8087bbb3-d125-4d57-b98e-774fd2a74485",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The simple HTML below displays differently in Firefox and WebKit-based browsers (I checked in Safari, Chrome and iPhone). In Firefox both border and text have the same color (#880000), but in Safari the text gets a bit lighter (as if it had some transparency applied to it). Can I somehow fix this (remove this transparency in Safari)? <html xmlns=\"http://www.w3.org/1999/xhtml\"> <head> <title></title> <style type=\"text/css\"> input:disabled{ border:solid 1px #880000; background-color:#ffffff; color:#880000; } </style> </head> <body> <form action=\"\"> <input type=\"text\" value=\"disabled input box\" disabled=\"disabled\"/> </form> </body> </html>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "168a999d-0fc7-4639-b552-e73420f46ba8",
   "metadata": {},
   "source": [
    "# Supprimer + de 2 carac à la suite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "271c16dd-430f-48dd-8427-1c03aae6de24",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "def reducing_incorrect_character_repeatation(text):\n",
    "    \"\"\"\n",
    "    This Function will reduce repeatition to two characters \n",
    "    for alphabets and to one character for punctuations.\n",
    "    \n",
    "    arguments:\n",
    "         input_text: \"text\" of type \"String\".\n",
    "         \n",
    "    return:\n",
    "        value: Finally formatted text with alphabets repeating to \n",
    "        two characters & punctuations limited to one repeatition \n",
    "        \n",
    "    Example:\n",
    "    Input : Realllllllllyyyyy,        Greeeeaaaatttt   !!!!?....;;;;:)\n",
    "    Output : Reallyy, Greeaatt !?.;:)\n",
    "    \n",
    "    \"\"\"\n",
    "    # Pattern matching for all case alphabets\n",
    "    Pattern_alpha = re.compile(r\"([A-Za-z])\\1{1,}\", re.DOTALL)\n",
    "    \n",
    "    # Limiting all the  repeatation to two characters.\n",
    "    Formatted_text = Pattern_alpha.sub(r\"\\1\\1\", text) \n",
    "    \n",
    "    # Pattern matching for all the punctuations that can occur\n",
    "    Pattern_Punct = re.compile(r'([.,/#!$%^&*?;:{}=_`~()+-])\\1{1,}')\n",
    "    \n",
    "    # Limiting punctuations in previously formatted string to only one.\n",
    "    Combined_Formatted = Pattern_Punct.sub(r'\\1', Formatted_text)\n",
    "    \n",
    "    # The below statement is replacing repeatation of spaces that occur more than two times with that of one occurrence.\n",
    "    Final_Formatted = re.sub(' {2,}',' ', Combined_Formatted)\n",
    "    return Final_Formatted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1924ee72-ea60-4f30-810d-36fb9e4f12e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\evari\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "df_questions['title'] = df_questions['title'].apply(lambda x: reducing_incorrect_character_repeatation(x)) \n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: reducing_incorrect_character_repeatation(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "195340c4-36de-4f01-a606-bf6f863be5f8",
   "metadata": {},
   "source": [
    "## 1.3.2 : Correction des mots mal orthographiés"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "bcacb49d-37b7-4ca8-9d29-11265c4f28fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def autocorrect(text):\n",
    "    words = token.tokenize(text)\n",
    "    \n",
    "    # Option 'fast' in place\n",
    "    spell = Speller(fast=True)\n",
    "    \n",
    "    words_correct = [spell(w) for w in words]\n",
    "    return ' '.join(map(str, words_correct)) # Return the text untokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "37ac7613-abe5-4dde-96c2-4c56b0ba794f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Trop gourmant en temps de calcul\n",
    "# Solution suffisante : supprimer les mots qui n'apparaissent pas X nombre de fois dans tous les documents\n",
    "# Temps dépensé pour le tout\n",
    "start = timeit.default_timer()\n",
    "\n",
    "# %%time\n",
    "\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: autocorrect(x)) \n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: autocorrect(x)) \n",
    "\n",
    "end = timeit.default_timer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "74c3dcbd-9723-4794-ab9a-fed4e15b4968",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  18981.779713800002\n"
     ]
    }
   ],
   "source": [
    "print('Time: ', end - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ec53cff3-f90f-4436-b3eb-66e624975d1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Takes 4.2 hours in this cas / i5 + RAM=16gigas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41c563a0-b404-4827-88bf-2f7673c42a2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a57e1c91-9997-4d08-abf8-8a47b5f90075",
   "metadata": {},
   "source": [
    "## 1.3.2 : Gestion des caractères en majuscule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5e7d5c4f-e16c-4b5d-bab5-dc9dbf0afd3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 305 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_questions['title'] = df_questions['title'].str.lower()\n",
    "df_questions['doc'] = df_questions['doc'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "513c6e50-1cc0-4397-acd1-76108aea453d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the simple html below displays differently in firefox and webkit-based browsers (i checked in safari, chrome and iphone).\\nin firefox both border and text have the same color (#880000), but in safari the text gets a bit lighter (as if it had some transparency applied to it).\\ncan i somehow fix this (remove this transparency in safari)?\\n\\n\\n<html xmlns=\"http:/ww.w3.org/1999/xhtml\">\\n <head>\\n <title></title>\\n <style type=\"text/css\">\\n input:disabled{\\n border:solid 1px #880000;\\n background-color:#ff;\\n color:#880000;\\n }\\n </style>\\n </head>\\n <body>\\n <form action=\"\">\\n <input type=\"text\" value=\"disabled input box\" disabled=\"disabled\"/>\\n </form>\\n </body>\\n</html>\\n\\n\\n\\n'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f86fa27-8aaa-4433-bac9-c1eeb4f95f73",
   "metadata": {},
   "source": [
    "# 1.4 : Suppression de caractères"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff387b60-5126-4f81-8244-cda62a79fa95",
   "metadata": {},
   "source": [
    "## 1.4.1 : Suppression des caractères spéciaux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6152dee4-9f37-401d-8edb-f593bb95c707",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_punctuation_and_number(text):\n",
    "    \"\"\"remove all punctuation and number\"\"\"\n",
    "    return text.translate(str.maketrans(\" \", \" \", charac)) \n",
    "\n",
    "\n",
    "def remove_non_alphabetical_character(text):\n",
    "    \"\"\"remove all non-alphabetical character\"\"\"\n",
    "    text = re.sub(\"[^a-z]+\", \" \", text) # remove all non-alphabetical character\n",
    "    text = re.sub(\"\\s+\", \" \", text) # remove whitespaces left after the last operation\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3715ec57-4854-49fe-b16a-84dc3e4161fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: remove_non_alphabetical_character(x)) \n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: remove_non_alphabetical_character(x)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c0e0663d-e462-4bad-8d8e-a6d0ed774684",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the simple html below displays differently in firefox and webkit based browsers i checked in safari chrome and iphone in firefox both border and text have the same color but in safari the text gets a bit lighter as if it had some transparency applied to it can i somehow fix this remove this transparency in safari html xmlns http ww w org xhtml head title title style type text css input disabled border solid px background color ff color style head body form action input type text value disabled input box disabled disabled form body html '"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49cd6d9e-6759-4bc3-81fa-90a3cade9c41",
   "metadata": {},
   "source": [
    "# 1.4.2 : Suppression des lettres isolées"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1b13f54d-0624-4107-86b4-5c47d0cb6d0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_single_letter(text):\n",
    "    \"\"\"remove single alphabetical character\"\"\"\n",
    "    text = re.sub(r\"\\b\\w{1}\\b\", \"\", text) # remove all single letter\n",
    "    text = re.sub(\"\\s+\", \" \", text) # remove whitespaces left after the last operation\n",
    "    text = text.strip(\" \")\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "70c24b00-7f97-4341-9524-fbf6c0a84b28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 14.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: remove_single_letter(x)) \n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: remove_single_letter(x)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "72f3bd71-23bd-43d0-85af-183d98a8cc6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the simple html below displays differently in firefox and webkit based browsers checked in safari chrome and iphone in firefox both border and text have the same color but in safari the text gets bit lighter as if it had some transparency applied to it can somehow fix this remove this transparency in safari html xmlns http www org xhtml head title title style type text css input disabled border solid px background color ffffff color style head body form action input type text value disabled input box disabled disabled form body html'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c1b8d34-f3be-426c-982c-ba368464b9d5",
   "metadata": {},
   "source": [
    "# 1.5 : Gestion des StopWords"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c0c4c69-c5b5-4552-ab15-dd17155e0d29",
   "metadata": {},
   "source": [
    "## 1.5.1 : Suppression des mots communs peu pertinents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2f992915-e7ba-495e-8e29-a221310771f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    \"\"\"remove common words in english by using nltk.corpus's list\"\"\"\n",
    "    words = token.tokenize(text)\n",
    "    filtered = [w for w in words if not w in stop_words]\n",
    "    \n",
    "    return ' '.join(map(str, filtered)) # Return the text untokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2582b1e9-0b9c-41cf-bab8-4ccc35fe7f75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 44.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: remove_stopwords(x))\n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: remove_stopwords(x)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "78d97cd4-24dc-412f-b728-a9aa12e9edfd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'simple html displays differently firefox webkit based browsers checked safari chrome iphone firefox border text color safari text gets bit lighter transparency applied somehow fix remove transparency safari html xmlns http www org xhtml head title title style type text css input disabled border solid px background color ffffff color style head body form action input type text value disabled input box disabled disabled form body html'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47d08787-2a87-4dd2-8cfd-6d82a2a79b93",
   "metadata": {},
   "source": [
    "## 1.5.2 : Suppression des adjectifs (ou autres mots jugés inutiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "69668a67-6dc1-40a9-a592-74eebe392477",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_by_tag(text, undesired_tag):\n",
    "    \"\"\"remove all words by using ntk tag (adjectives, verbs, etc.)\"\"\"\n",
    "    words = token.tokenize(text) # Tokenize each words\n",
    "    words_tagged = nltk.pos_tag(tokens=words, tagset=None, lang='eng') # Tag each words and return a list of tuples (e.g. (\"have\", \"VB\"))\n",
    "    filtered = [w[0] for w in words_tagged if w[1] not in undesired_tag] # Select all words that don't have the undesired tags\n",
    "    \n",
    "    return ' '.join(map(str, filtered)) # Return the text untokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "a1c293ad-62c3-4e11-9efa-64ec895c1a2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 24min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: remove_by_tag(x, adjective_tag_list))\n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: remove_by_tag(x, adjective_tag_list))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "25411eb4-f8aa-4baf-b1d6-f40768273958",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'simple html displays differently firefox webkit based browsers checked chrome iphone firefox border text color safari text gets bit transparency applied remove transparency safari xmlns http www org xhtml head title style type text css input disabled border px background color ffffff color style head body action input type text value disabled input box disabled form body html'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1098c6a-3f76-4236-80b6-92c47502014d",
   "metadata": {},
   "source": [
    "# 1.6 : Stemming | Lemmatizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "51031014-bb84-4979-826a-349fe1548d45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "program  :  program\n",
      "programs  :  program\n",
      "programer  :  program\n",
      "programing  :  program\n",
      "programers  :  program\n"
     ]
    }
   ],
   "source": [
    "# Method testing\n",
    "words = [\"program\", \"programs\", \"programer\", \"programing\", \"programers\"]\n",
    "  \n",
    "for w in words:\n",
    "    print(w, \" : \", stemmer.stem(w))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "264ffc9d-cab0-4dec-9f7d-dc9ec867394d",
   "metadata": {},
   "source": [
    "## 1.6.1 : Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a77bf325-37d2-4027-925c-8dbbdd2a91cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def stem_text(text):\n",
    "    \"\"\"Stem the text\"\"\"\n",
    "    words = nltk.word_tokenize(text) # tokenize the text then return a list of tuple (token, nltk_tag)\n",
    "    stem_text = []\n",
    "    for word in words:\n",
    "        stem_text.append(stemmer.stem(word)) # Stem each words\n",
    "    return \" \".join(stem_text) # Return the text untokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "98206cb6-e5e9-4c96-8113-c248274e5589",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# df_questions['title'] = df_questions['title'].apply(lambda x: stem_text(x)) \n",
    "# df_questions['doc'] = df_questions['doc'].apply(lambda x: stem_text(x)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4502cc11-91aa-4da5-a7cb-333803733c0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec5adec5-bad2-421f-b93d-3bd28e3a65f2",
   "metadata": {},
   "source": [
    "## 5.2 : Lemmatizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "8caf6b5c-7509-47df-ac3f-b561fd2eea3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "insanities\n",
      "insanity\n",
      "are\n",
      "be\n"
     ]
    }
   ],
   "source": [
    "# Small test\n",
    "print(lemmatizer.lemmatize(\"insanities\", \"v\"))\n",
    "print(lemmatizer.lemmatize(\"insanities\", \"n\"))  \n",
    "print(lemmatizer.lemmatize(\"are\"))\n",
    "print(lemmatizer.lemmatize(\"are\", \"v\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "3eb1bac2-7c6d-479e-8638-17b4e71773ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_text(text):\n",
    "    \"\"\"Lemmatize the text by using tag \"\"\"\n",
    "    \n",
    "    tokens_tagged = nltk.pos_tag(nltk.word_tokenize(text))  # tokenize the text then return a list of tuple (token, nltk_tag)\n",
    "    lemmatized_text = []\n",
    "    for word, tag in tokens_tagged:\n",
    "        if tag.startswith('J'):\n",
    "            lemmatized_text.append(lemmatizer.lemmatize(word,'a')) # Lemmatisze adjectives. Not doing anything since we remove all adjective\n",
    "        elif tag.startswith('V'):\n",
    "            lemmatized_text.append(lemmatizer.lemmatize(word,'v')) # Lemmatisze verbs\n",
    "        elif tag.startswith('N'):\n",
    "            lemmatized_text.append(lemmatizer.lemmatize(word,'n')) # Lemmatisze nouns\n",
    "        elif tag.startswith('R'):\n",
    "            lemmatized_text.append(lemmatizer.lemmatize(word,'r')) # Lemmatisze adverbs\n",
    "        else:\n",
    "            lemmatized_text.append(lemmatizer.lemmatize(word)) # If no tags has been found, perform a non specific lemmatization\n",
    "    return \" \".join(lemmatized_text) # Return the text untokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "1884a51c-766c-4225-8f79-4411299b9174",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 20min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_questions['title'] = df_questions['title'].apply(lambda x: lemmatize_text(x)) \n",
    "df_questions['doc'] = df_questions['doc'].apply(lambda x: lemmatize_text(x)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "2953392a-abcb-4e38-b096-9fc4a4ccfb96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'simple html display differently firefox webkit base browser check chrome iphone firefox border text color safari text get bit transparency apply remove transparency safari xmlns http www org xhtml head title style type text cs input disable border px background color ffffff color style head body action input type text value disable input box disable form body html'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d9c95bd-45fe-4ec9-955d-fef906c55b26",
   "metadata": {},
   "source": [
    "# Le tout en une seule fonction :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "457d2e17-c003-4a15-a858-2d185045e31f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_questions_V2 = df_questions.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9247aa32-964d-4b27-8369-024cce63c5c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def prepared_text(df_questions_V2):\n",
    "    \n",
    "#     # Noise cleaning / html etc\n",
    "#     df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: clean_text(x))\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: clean_text(x))\n",
    "    \n",
    "#     # Deal with contractions : do and do not etc\n",
    "#     df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: expand_contractions(x))\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: expand_contractions(x))\n",
    "    \n",
    "#     # Word grammatical correction : cost is quite high, ~4hours in this case\n",
    "#     # Might lead to deleting informations like c, r and c++\n",
    "#     df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: autocorrect(x))\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: autocorrect(x))\n",
    "    \n",
    "#     # Lower the words\n",
    "#     df_questions_V2['title'] = df_questions_V2['title'].str.lower()\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].str.lower()\n",
    "    \n",
    "#     # Remove special characters, ex : pd with info c++\n",
    "#     df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: remove_non_alphabetical_character(x))\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: remove_non_alphabetical_character(x))\n",
    "    \n",
    "#     # Choice : Can lead to delete info like c, r, c++ that are pertinent info\n",
    "#     #df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: remove_single_letter(x))\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: remove_single_letter(x))\n",
    "    \n",
    "#     # Remove all stop words\n",
    "#     df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: remove_stopwords(x))\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: remove_stopwords(x))\n",
    "    \n",
    "#     # Remove adjectives that do not bring anything here\n",
    "#     df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: remove_by_tag(x, adjective_tag_list))\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: remove_by_tag(x, adjective_tag_list))\n",
    "    \n",
    "#     # Result unprecised, leads to worse predictions\n",
    "#     # df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: stem_text(x)) \n",
    "#     # df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: stem_text(x)) \n",
    "    \n",
    "#     # Lemmatize words\n",
    "#     df_questions_V2['title'] = df_questions_V2['title'].apply(lambda x: lemmatize_text(x))\n",
    "#     df_questions_V2['doc'] = df_questions_V2['doc'].apply(lambda x: lemmatize_text(x))\n",
    "    \n",
    "#     # Feature engineering : 2 texts in one\n",
    "#     df_doc_test['text'] = df_doc_test['title'] + ' ' + df_doc_test['doc']\n",
    "    \n",
    "#     return df_questions_V2 # Format !!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f57a67f-5f0e-44d4-b4c2-3586abda6ee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_questions['doc'][N_doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb57cc4b-5a5e-4e7d-bd1a-c4a8a6a82e05",
   "metadata": {},
   "source": [
    "# 6 : Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "bd164b03-a412-45b5-bcf9-7329a03c53fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_questions['text'] = df_questions['title'] + ' ' + df_questions['doc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "41d0216e-f870-4c8c-a343-ca7ea8cf86dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>doc</th>\n",
       "      <th>title</th>\n",
       "      <th>tags</th>\n",
       "      <th>creation_date</th>\n",
       "      <th>score</th>\n",
       "      <th>views</th>\n",
       "      <th>answers</th>\n",
       "      <th>comments</th>\n",
       "      <th>favorites</th>\n",
       "      <th>last_activity_date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>9056957</td>\n",
       "      <td>python people initialize class attribute two w...</td>\n",
       "      <td>way class variable python</td>\n",
       "      <td>&lt;python&gt;&lt;class&gt;&lt;attributes&gt;</td>\n",
       "      <td>2012-01-29 21:26:39</td>\n",
       "      <td>296</td>\n",
       "      <td>372888</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>105</td>\n",
       "      <td>2021-01-25 13:17:29</td>\n",
       "      <td>way class variable python python people initia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>52013133</td>\n",
       "      <td>use webpack create config file try use htmlweb...</td>\n",
       "      <td>entrypoint index html use htmlwebpackplugin</td>\n",
       "      <td>&lt;node.js&gt;&lt;reactjs&gt;&lt;webpack&gt;&lt;html-webpack-plugin&gt;</td>\n",
       "      <td>2018-08-25 00:32:17</td>\n",
       "      <td>31</td>\n",
       "      <td>33607</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2021-07-18 06:19:28</td>\n",
       "      <td>entrypoint index html use htmlwebpackplugin us...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>261963</td>\n",
       "      <td>notice math operator num way literate value num</td>\n",
       "      <td>literate num</td>\n",
       "      <td>&lt;c++&gt;&lt;enums&gt;</td>\n",
       "      <td>2008-11-04 13:55:29</td>\n",
       "      <td>354</td>\n",
       "      <td>374899</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>67</td>\n",
       "      <td>2021-05-23 11:33:10</td>\n",
       "      <td>literate num notice math operator num way lite...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>60684868</td>\n",
       "      <td>currently try use pretrained model provide dec...</td>\n",
       "      <td>decker image use compile work cpu</td>\n",
       "      <td>&lt;docker&gt;&lt;tensorflow&gt;</td>\n",
       "      <td>2020-03-14 16:25:39</td>\n",
       "      <td>1</td>\n",
       "      <td>225</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-01-04 03:48:09</td>\n",
       "      <td>decker image use compile work cpu currently tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>35136616</td>\n",
       "      <td>can not figure case loss application write ele...</td>\n",
       "      <td>handle loss three j</td>\n",
       "      <td>&lt;javascript&gt;&lt;three.js&gt;&lt;webgl&gt;&lt;electron&gt;</td>\n",
       "      <td>2016-02-01 17:26:01</td>\n",
       "      <td>6</td>\n",
       "      <td>412</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-03-18 17:02:54</td>\n",
       "      <td>handle loss three j can not figure case loss a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0        id                                                doc  \\\n",
       "0           0   9056957  python people initialize class attribute two w...   \n",
       "1           1  52013133  use webpack create config file try use htmlweb...   \n",
       "2           2    261963    notice math operator num way literate value num   \n",
       "3           3  60684868  currently try use pretrained model provide dec...   \n",
       "4           4  35136616  can not figure case loss application write ele...   \n",
       "\n",
       "                                         title  \\\n",
       "0                    way class variable python   \n",
       "1  entrypoint index html use htmlwebpackplugin   \n",
       "2                                 literate num   \n",
       "3            decker image use compile work cpu   \n",
       "4                          handle loss three j   \n",
       "\n",
       "                                               tags        creation_date  \\\n",
       "0                       <python><class><attributes>  2012-01-29 21:26:39   \n",
       "1  <node.js><reactjs><webpack><html-webpack-plugin>  2018-08-25 00:32:17   \n",
       "2                                      <c++><enums>  2008-11-04 13:55:29   \n",
       "3                              <docker><tensorflow>  2020-03-14 16:25:39   \n",
       "4           <javascript><three.js><webgl><electron>  2016-02-01 17:26:01   \n",
       "\n",
       "   score   views  answers  comments  favorites   last_activity_date  \\\n",
       "0    296  372888        2         1        105  2021-01-25 13:17:29   \n",
       "1     31   33607        5         2          7  2021-07-18 06:19:28   \n",
       "2    354  374899       24         5         67  2021-05-23 11:33:10   \n",
       "3      1     225        1         5          1  2021-01-04 03:48:09   \n",
       "4      6     412        1         0          1  2021-03-18 17:02:54   \n",
       "\n",
       "                                                text  \n",
       "0  way class variable python python people initia...  \n",
       "1  entrypoint index html use htmlwebpackplugin us...  \n",
       "2  literate num notice math operator num way lite...  \n",
       "3  decker image use compile work cpu currently tr...  \n",
       "4  handle loss three j can not figure case loss a...  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_questions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "23e557db-c596-43eb-a652-6e4db745bb22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  22051.9593666\n"
     ]
    }
   ],
   "source": [
    "# Temps dépensé pour le tout\n",
    "stop_all = timeit.default_timer()\n",
    "print('Time: ', stop_all - start_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "b4d1ec2e-1e4b-4b52-bc04-dc48f42e46bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99986, 13)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_questions.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abbbd964-982d-4ce3-9d15-8f1158f8f9f9",
   "metadata": {},
   "source": [
    "# 7 : Enregistrement du dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "c6d4efc5-1725-42f6-9a37-49fa3ea17411",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5.08 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_questions.to_csv('df_questions_cleaned.csv', encoding='utf-8', errors='surrogatepass')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "089db375-0ca6-4fae-a121-0f0993728322",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}