P5_Packages.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da561cfd-0532-4bbe-a56f-70d703d29cd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import warnings\n",
    "\n",
    "# warnings.filterwarnings(action='always')\n",
    "# warnings.filterwarnings(\"ignore\",category=DeprecationWarning)\n",
    "# warnings.filterwarnings(\"ignore\",category=ResourceWarning)\n",
    "\n",
    "import sys\n",
    "import warnings\n",
    "\n",
    "if not sys.warnoptions:\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    \n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c28fe7a0-7280-4ea4-b6c3-8f8b5f16fa39",
   "metadata": {},
   "source": [
    "# All required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3152dec-9f02-4f14-a4e9-56cbc7e7cdbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install required packages\n",
    "!pip install keyboard\n",
    "!pip install bs4\n",
    "!pip install autocorrect\n",
    "!pip install wordcloud\n",
    "#visualisation LDA models\n",
    "!pip install pyLDAvis "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f62a4e72-0ed2-4e3b-846a-598c8a465d46",
   "metadata": {},
   "source": [
    "# Generic librairies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb4e4293-afc8-4e4e-8cc0-23520673c0a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# generic librairies\n",
    "import time as time\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import re\n",
    "import gc\n",
    "import json # Pour gérer ce format, utile pour la colonne de texte\n",
    "import ast\n",
    "import timeit # Obtenir info sur temps de calcul\n",
    "import time # Temps calcul\n",
    "import pickle # Save and open ML models\n",
    "import itertools # to merge lists\n",
    "import joblib # save and open model like pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ec81f58-5170-4247-ab25-b4392769024e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pour faire appel au clavier et input/output utilisateur\n",
    "import sys\n",
    "import keyboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "882bfb76-2a7c-4fa8-81b0-25c1a802ab6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9039a87d-9a43-4c9b-a256-72f404f4e348",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime #Gérer les objet de temps/date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62c236b7-1849-4204-9c6f-cb859df4981e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pour gerer les fichiers / en ouverture\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f4c22ec-dd99-470b-a674-9ea674726ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Package to flatten a list\n",
    "# get rid of [[ ]]\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20f5f61f-267f-4ea7-9881-55d3592bc670",
   "metadata": {},
   "source": [
    "# Packages | Maths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c0ff42d-92e3-4d3d-a87c-246767813936",
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import radians, cos, sin, arcsin, sqrt\n",
    "import math\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "from math import pi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8edade0-e16f-4a52-afa4-691f44eaf9ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm\n",
    "import pylab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d14f11d-ad05-4c93-834a-dd5ecbd2acfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e807dd1-e9a5-495d-b272-18b25f2436bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "# nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a270cad6-ca81-4f20-8af6-21fb27b03367",
   "metadata": {},
   "source": [
    "# Packages | text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6df2375-fae7-4ef1-bf23-723d226ca3b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim.matutils import corpus2csc\n",
    "from gensim.models import Phrases\n",
    "from gensim.models.phrases import Phraser\n",
    "from gensim.utils import simple_preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13113d29-5b4d-425e-8ec3-aa75bdeb98f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install contractions\n",
    "from contractions import CONTRACTION_MAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "240762b0-f136-4d05-8227-de0555706984",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Text librairies\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "import requests\n",
    "from nltk.tokenize import ToktokTokenizer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tag.util import untag\n",
    "from autocorrect import Speller\n",
    "import contractions\n",
    "# import pycontractions # Alternative better package for removing contractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "450e8a46-c601-401b-9bee-4f252674213c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import unicodedata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b31313b4-b1cd-49b3-9045-fb8b0b4c95e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer = ToktokTokenizer()\n",
    "# stopword_list = nltk.corpus.stopwords.words('english')\n",
    "# stopword_list.remove('no')\n",
    "# stopword_list.remove('not')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cea31f85-e2f7-49c0-888a-8dec96c5b81e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the wordcloud library\n",
    "from wordcloud import WordCloud# Join the different processed titles together."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39e8dde3-8fa3-4c8a-aedf-868ec03b58e2",
   "metadata": {},
   "source": [
    "# Machine Learning for text data : Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e3d5a58-6d52-4836-ba09-cbb275dc814e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Classif random forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Classif SVM\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "# To be able to do OneVsRest classifications\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "\n",
    "# Naive Bayes Model\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b4bcafa-1db1-4ff7-b3c1-427abf67691b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logistic Regression Model for classification\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# Xgboost classification model\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "# Module to binarize data\n",
    "from sklearn.preprocessing import Binarizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "361f6ecd-f4a5-424a-8c9a-de0f669b69a7",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94fccbde-8051-4710-887e-c8179be57d3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform feature selection using a variance threshold\n",
    "from sklearn.feature_selection import VarianceThreshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd6e8ad9-eee1-4ea8-8040-2e36eaa64063",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import FunctionTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95608331-55c3-406c-b57b-e66156d47d8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Method to apply imputation to missing data\n",
    "from sklearn.impute import SimpleImputer\n",
    "# KNN imputation model for missing data\n",
    "from sklearn.impute import KNNImputer "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "673b3b7a-bfa5-4c09-a3c7-8b626d111597",
   "metadata": {},
   "source": [
    "# Packages sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "144ac175-fa81-4a49-9254-f71f8459a6db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Package to split data into train/test\n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn import model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "435d03d2-4cfe-4ae5-87f7-63db1b13ef24",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Classes required to have a cross validation respecting classes distributions\n",
    "# concerned : StratifiedKFold\n",
    "from sklearn.model_selection import StratifiedKFold \n",
    "\n",
    "# Séparations en folds : méthode classique\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "# Scores, errors of algorithms\n",
    "from sklearn import neighbors, metrics\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58bd1bc6-03dc-4459-bd36-ff2ad3bdb3ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find out about multicolinearity\n",
    "from statsmodels.stats.outliers_influence import variance_inflation_factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab8c9252-b513-4da7-ba52-9257eb0725d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pipeline method\n",
    "from sklearn.pipeline import make_pipeline \n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "# Function used inside pipeline\n",
    "from sklearn.compose import TransformedTargetRegressor\n",
    "\n",
    "# To get optimal hyperparameters\n",
    "from sklearn.model_selection import GridSearchCV \n",
    "from sklearn.model_selection import RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "976cde4e-d11d-46b8-9d4a-3fce0a216ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To deal with text data\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.preprocessing import MultiLabelBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "52fd02dc-cefd-4469-a182-0bc3aaf3533d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score, \\\n",
    "    recall_score, confusion_matrix, classification_report, \\\n",
    "    accuracy_score, f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f76d89a0-340b-4fdc-80ac-08ea625fd07f",
   "metadata": {},
   "source": [
    "# Machine Learning Models for text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "892d574c-ecd8-4b7f-971d-bcc55fd636bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyLDAvis\n",
    "# import pyLDAvis.gensim_models as gensimvis\n",
    "pyLDAvis.enable_notebook()\n",
    "\n",
    "import pyLDAvis.gensim as gensimvis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdfaded1-ee45-4701-9751-f1f7cdc88c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unsupervised models\n",
    "from sklearn.decomposition import NMF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "483ee2af-0fe6-4b90-bd32-83f256eec114",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Packages related to gensim\n",
    "import gensim\n",
    "from gensim.test.utils import common_corpus, common_dictionary\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "from gensim import corpora, models\n",
    "# Unsupervised model : NMF\n",
    "from gensim.models import Nmf\n",
    "\n",
    "import gensim\n",
    "from pprint import pprint# number of topics\n",
    "from gensim.test.utils import common_corpus, common_dictionary\n",
    "from gensim.models.coherencemodel import CoherenceModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c924b73-b747-4c38-a645-44a38827d354",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d513a210-7890-42fd-aa20-01f94cfd1192",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import random \n",
    "from random import sample\n",
    "from collections import Counter\n",
    "from matplotlib.ticker import MaxNLocator\n",
    "from pprint import pprint# number of topics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20730ce5-4ac2-4265-8da7-3b47c1b605a7",
   "metadata": {},
   "source": [
    "# Post-Processing - Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b62da1fd-7979-4bf9-9055-ae8996986901",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # installation de yellowbrick nécessaire\n",
    "# !pip install --user yellowbrick"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58d123a6-85c8-4297-a658-2e6d18e295e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Pour visualiser les elbows en graphes\n",
    "# from yellowbrick.cluster import KElbowVisualizer, SilhouetteVisualizer\n",
    "\n",
    "# # PCA decomposition\n",
    "# from sklearn import decomposition\n",
    "\n",
    "# # Scores\n",
    "# from sklearn.metrics import silhouette_score \n",
    "# from sklearn.metrics import calinski_harabasz_score\n",
    "\n",
    "# # Davies Bouldin score for K means\n",
    "# from sklearn.metrics import davies_bouldin_score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3306ac4d-292f-4fe5-8825-3650a1243078",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Inter cluster distance map\n",
    "# from yellowbrick.cluster import InterclusterDistance\n",
    "\n",
    "# # Graphe pour les centroides des clusters\n",
    "# from pandas.plotting import parallel_coordinates\n",
    "\n",
    "# from matplotlib.collections import LineCollection\n",
    "# from scipy.cluster.hierarchy import dendrogram\n",
    "\n",
    "# # Rand score\n",
    "# from sklearn.metrics.cluster import adjusted_rand_score\n",
    "\n",
    "# # Mutual score\n",
    "# from sklearn.metrics.cluster import adjusted_mutual_info_score\n",
    "\n",
    "# # Pour PCA\n",
    "# from sklearn import (manifold, datasets, decomposition, ensemble,discriminant_analysis, random_projection)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}