text scoring experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffe72fd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!conda install openpyxl -y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d9af9626",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>CASE</th>\n",
       "      <th>Q4 ANSWER</th>\n",
       "      <th>Q4 Score</th>\n",
       "      <th>Q9 ANSWER</th>\n",
       "      <th>Q9 Score</th>\n",
       "      <th>Score 1-4 Q4</th>\n",
       "      <th>Ashray Score</th>\n",
       "      <th>Score 1-4 Q9</th>\n",
       "      <th>Ashray Score.1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>Yes, offering the discounts will help increase...</td>\n",
       "      <td>44</td>\n",
       "      <td>In a normal scenario, the client would pay 100...</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>It depends. If they want to promote the use of...</td>\n",
       "      <td>33</td>\n",
       "      <td>If for every $10 increase the possibility of a...</td>\n",
       "      <td>78</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>the introduction is very profitable from Vivit...</td>\n",
       "      <td>0</td>\n",
       "      <td>the increase in premiums corresponds to an inc...</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>To decide whether this is a good idea, I would...</td>\n",
       "      <td>0</td>\n",
       "      <td>The expected cost increase by 150$ per opt-out...</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>Ich würde empfehlen das Angebot auch den beste...</td>\n",
       "      <td>6</td>\n",
       "      <td>`-</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0     CASE                                          Q4 ANSWER  \\\n",
       "0         NaN  Case 2a  Yes, offering the discounts will help increase...   \n",
       "1         NaN  Case 2a  It depends. If they want to promote the use of...   \n",
       "2         NaN  Case 2a  the introduction is very profitable from Vivit...   \n",
       "3         NaN  Case 2a  To decide whether this is a good idea, I would...   \n",
       "4         NaN  Case 2a  Ich würde empfehlen das Angebot auch den beste...   \n",
       "\n",
       "   Q4 Score                                          Q9 ANSWER  Q9 Score  \\\n",
       "0        44  In a normal scenario, the client would pay 100...        44   \n",
       "1        33  If for every $10 increase the possibility of a...        78   \n",
       "2         0  the increase in premiums corresponds to an inc...        33   \n",
       "3         0  The expected cost increase by 150$ per opt-out...        44   \n",
       "4         6                                                 `-         0   \n",
       "\n",
       "   Score 1-4 Q4  Ashray Score  Score 1-4 Q9  Ashray Score.1  \n",
       "0             2           NaN           2.0             2.0  \n",
       "1             2           NaN           4.0             2.0  \n",
       "2             1           NaN           2.0             4.0  \n",
       "3             1           NaN           2.0             1.0  \n",
       "4             1           NaN           1.0             1.0  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data = pd.read_excel (r'data.xlsx') \n",
    "df = pd.DataFrame(data)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e0ed3839",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'CASE', 'Q4 ANSWER', 'Q4 Score', 'Q9 ANSWER', 'Q9 Score',\n",
       "       'Score 1-4 Q4', 'Ashray Score', 'Score 1-4 Q9', 'Ashray Score.1'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8e399f51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q4 ANSWER</th>\n",
       "      <th>Q4 Score</th>\n",
       "      <th>Score 1-4 Q4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yes, offering the discounts will help increase...</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>It depends. If they want to promote the use of...</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the introduction is very profitable from Vivit...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>To decide whether this is a good idea, I would...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ich würde empfehlen das Angebot auch den beste...</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Q4 ANSWER  Q4 Score  Score 1-4 Q4\n",
       "0  Yes, offering the discounts will help increase...        44             2\n",
       "1  It depends. If they want to promote the use of...        33             2\n",
       "2  the introduction is very profitable from Vivit...         0             1\n",
       "3  To decide whether this is a good idea, I would...         0             1\n",
       "4  Ich würde empfehlen das Angebot auch den beste...         6             1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q4_df = df[[\"Q4 ANSWER\",\"Q4 Score\",\"Score 1-4 Q4\"]]\n",
    "q4_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "252276c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q9 ANSWER</th>\n",
       "      <th>Q9 Score</th>\n",
       "      <th>Score 1-4 Q9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>In a normal scenario, the client would pay 100...</td>\n",
       "      <td>44</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>If for every $10 increase the possibility of a...</td>\n",
       "      <td>78</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the increase in premiums corresponds to an inc...</td>\n",
       "      <td>33</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The expected cost increase by 150$ per opt-out...</td>\n",
       "      <td>44</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>`-</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Q9 ANSWER  Q9 Score  Score 1-4 Q9\n",
       "0  In a normal scenario, the client would pay 100...        44           2.0\n",
       "1  If for every $10 increase the possibility of a...        78           4.0\n",
       "2  the increase in premiums corresponds to an inc...        33           2.0\n",
       "3  The expected cost increase by 150$ per opt-out...        44           2.0\n",
       "4                                                 `-         0           1.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q9_df = df[[\"Q9 ANSWER\",\"Q9 Score\",\"Score 1-4 Q9\"]]\n",
    "q9_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2beb8e0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4_df = q4_df.rename(columns = {\"Q4 ANSWER\":\"answer\",\"Q4 Score\":\"score_100\",\"Score 1-4 Q4\":\"score_4\"})\n",
    "q9_df = q9_df.rename(columns = {\"Q9 ANSWER\":\"answer\",\"Q9 Score\":\"score_100\",\"Score 1-4 Q9\":\"score_4\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "382af11e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>score_100</th>\n",
       "      <th>score_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yes, offering the discounts will help increase...</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>It depends. If they want to promote the use of...</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the introduction is very profitable from Vivit...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>To decide whether this is a good idea, I would...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ich würde empfehlen das Angebot auch den beste...</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              answer  score_100  score_4\n",
       "0  Yes, offering the discounts will help increase...         44        2\n",
       "1  It depends. If they want to promote the use of...         33        2\n",
       "2  the introduction is very profitable from Vivit...          0        1\n",
       "3  To decide whether this is a good idea, I would...          0        1\n",
       "4  Ich würde empfehlen das Angebot auch den beste...          6        1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q4_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "10e10eb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4_df = q4_df[q4_df.answer != \"`-\"]\n",
    "q9_df = q9_df[q9_df.answer != \"`-\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "27141063",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/uday/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "st = SnowballStemmer('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "beeb0127",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to clean data\n",
    "def clean_data(df, col, clean_col):\n",
    "\n",
    "    # change to lower and remove spaces on either side\n",
    "    df[clean_col] = df[col].apply(lambda x: x.lower().strip())\n",
    "\n",
    "    # remove extra spaces in between\n",
    "    df[clean_col] = df[clean_col].apply(lambda x: re.sub(' +', ' ', x))\n",
    "\n",
    "    # remove punctuation\n",
    "    df[clean_col] = df[clean_col].apply(lambda x: re.sub('[^a-zA-Z]', ' ', x))\n",
    "\n",
    "    # remove stopwords and get the stem\n",
    "    df[clean_col] = df[clean_col].apply(lambda x: ' '.join(st.stem(text) for text in x.split() if text not in stop_words))\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "181c0c95",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4_df = clean_data(q4_df, 'answer', 'answer')\n",
    "q9_df = clean_data(q9_df, 'answer', 'answer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a31843b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import numpy as np\n",
    "v = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4fb7b9a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/uday/miniforge3/envs/rasatest/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>custom</td>\n",
       "      <td>0.096410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>986</th>\n",
       "      <td>would</td>\n",
       "      <td>0.074586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>discount</td>\n",
       "      <td>0.070932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>exist</td>\n",
       "      <td>0.064906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>offer</td>\n",
       "      <td>0.064096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>689</th>\n",
       "      <td>profit</td>\n",
       "      <td>0.053185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>claim</td>\n",
       "      <td>0.053035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>445</th>\n",
       "      <td>increas</td>\n",
       "      <td>0.049139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>vivita</td>\n",
       "      <td>0.047082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>decreas</td>\n",
       "      <td>0.041537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>also</td>\n",
       "      <td>0.039935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>could</td>\n",
       "      <td>0.036535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>897</th>\n",
       "      <td>tracker</td>\n",
       "      <td>0.034860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>fit</td>\n",
       "      <td>0.034591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>client</td>\n",
       "      <td>0.033300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>likelihood</td>\n",
       "      <td>0.032788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>990</th>\n",
       "      <td>yes</td>\n",
       "      <td>0.032686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>opt</td>\n",
       "      <td>0.031673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>cost</td>\n",
       "      <td>0.031614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>new</td>\n",
       "      <td>0.031187</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           term    weight\n",
       "200      custom  0.096410\n",
       "986       would  0.074586\n",
       "246    discount  0.070932\n",
       "323       exist  0.064906\n",
       "605       offer  0.064096\n",
       "689      profit  0.053185\n",
       "149       claim  0.053035\n",
       "445     increas  0.049139\n",
       "946      vivita  0.047082\n",
       "219     decreas  0.041537\n",
       "38         also  0.039935\n",
       "190       could  0.036535\n",
       "897     tracker  0.034860\n",
       "356         fit  0.034591\n",
       "151      client  0.033300\n",
       "507  likelihood  0.032788\n",
       "990         yes  0.032686\n",
       "615         opt  0.031673\n",
       "188        cost  0.031614\n",
       "591         new  0.031187"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = v.fit_transform(q4_df['answer'])\n",
    "q4_df['tfidf'] = list(x.toarray())\n",
    "\n",
    "weights = np.asarray(x.mean(axis=0)).ravel().tolist()\n",
    "weights_df = pd.DataFrame({'term': v.get_feature_names(), 'weight': weights})\n",
    "weights_df.sort_values(by='weight', ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "64eaafe4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/uday/miniforge3/envs/rasatest/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>increas</td>\n",
       "      <td>0.161797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>claim</td>\n",
       "      <td>0.092335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>premium</td>\n",
       "      <td>0.086447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>opt</td>\n",
       "      <td>0.079508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345</th>\n",
       "      <td>profit</td>\n",
       "      <td>0.079015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>would</td>\n",
       "      <td>0.072301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>cost</td>\n",
       "      <td>0.069526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>revenu</td>\n",
       "      <td>0.056382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>higher</td>\n",
       "      <td>0.055100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>decreas</td>\n",
       "      <td>0.053639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>probabl</td>\n",
       "      <td>0.052311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>dollar</td>\n",
       "      <td>0.044406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>price</td>\n",
       "      <td>0.041443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320</th>\n",
       "      <td>per</td>\n",
       "      <td>0.041049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>expect</td>\n",
       "      <td>0.039973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>therefor</td>\n",
       "      <td>0.035920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>group</td>\n",
       "      <td>0.035178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>likelihood</td>\n",
       "      <td>0.032290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>pool</td>\n",
       "      <td>0.030715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>number</td>\n",
       "      <td>0.028405</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           term    weight\n",
       "211     increas  0.161797\n",
       "62        claim  0.092335\n",
       "335     premium  0.086447\n",
       "302         opt  0.079508\n",
       "345      profit  0.079015\n",
       "489       would  0.072301\n",
       "90         cost  0.069526\n",
       "383      revenu  0.056382\n",
       "197      higher  0.055100\n",
       "108     decreas  0.053639\n",
       "343     probabl  0.052311\n",
       "125      dollar  0.044406\n",
       "339       price  0.041443\n",
       "320         per  0.041049\n",
       "155      expect  0.039973\n",
       "443    therefor  0.035920\n",
       "190       group  0.035178\n",
       "245  likelihood  0.032290\n",
       "329        pool  0.030715\n",
       "293      number  0.028405"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = v.fit_transform(q9_df['answer'])\n",
    "q9_df['tfidf'] = list(x.toarray())\n",
    "\n",
    "weights = np.asarray(x.mean(axis=0)).ravel().tolist()\n",
    "weights_df = pd.DataFrame({'term': v.get_feature_names(), 'weight': weights})\n",
    "weights_df.sort_values(by='weight', ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ffeea40d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>score_100</th>\n",
       "      <th>score_4</th>\n",
       "      <th>tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>normal scenario client would pay would die rec...</td>\n",
       "      <td>44</td>\n",
       "      <td>2.0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>everi increas possibl claim increas eventu cos...</td>\n",
       "      <td>78</td>\n",
       "      <td>4.0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>increas premium correspond increas meanwhil in...</td>\n",
       "      <td>33</td>\n",
       "      <td>2.0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>expect cost increas per opt wherea revenu increas</td>\n",
       "      <td>44</td>\n",
       "      <td>2.0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>probabl claim increas possibl cost also increa...</td>\n",
       "      <td>67</td>\n",
       "      <td>3.0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              answer  score_100  score_4  \\\n",
       "0  normal scenario client would pay would die rec...         44      2.0   \n",
       "1  everi increas possibl claim increas eventu cos...         78      4.0   \n",
       "2  increas premium correspond increas meanwhil in...         33      2.0   \n",
       "3  expect cost increas per opt wherea revenu increas         44      2.0   \n",
       "6  probabl claim increas possibl cost also increa...         67      3.0   \n",
       "\n",
       "                                               tfidf  \n",
       "0  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "1  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "2  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "3  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "6  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q9_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c180bbe5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from sklearn import preprocessing, svm\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import LogisticRegressionCV     \n",
    "from sklearn import linear_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8bcf015f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(176,) (20,) (176,) (20,)\n",
      "108     0\n",
      "14      0\n",
      "43     44\n",
      "162     0\n",
      "263    25\n",
      "206    42\n",
      "248    50\n",
      "191     0\n",
      "95     56\n",
      "23     67\n",
      "9      11\n",
      "7      78\n",
      "192    67\n",
      "246    83\n",
      "262     0\n",
      "278     0\n",
      "190    67\n",
      "77     78\n",
      "57     56\n",
      "214     0\n",
      "Name: score_100, dtype: int64\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# insistent our storage \n",
    "bench_mark = defaultdict(list)\n",
    "\n",
    "# split data random state 0 and test_size 0.25 default as you did not give the test_size\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(q4_df['answer'], q4_df['score_4'], random_state = 0, test_size = 0.1)\n",
    "# X_train, X_test, y_train, y_test = train_test_split(q4_df['answer'], q4_df['score_4'], random_state = 0, test_size = 0.1)\n",
    "X_train, X_test, y_train, y_test = train_test_split(q9_df['answer'], q9_df['score_100'], random_state = 0, test_size = 0.1)\n",
    "# X_train, X_test, y_train, y_test = train_test_split(q9_df['answer'], q9_df['score_100'], random_state = 0, test_size = 0.1)\n",
    "\n",
    "\n",
    "#get the transformer params\n",
    "tfidf_params = dict(sublinear_tf= True, \n",
    "                       min_df = 1,\n",
    "                       norm= 'l2', \n",
    "                       ngram_range= (1,2), \n",
    "                       stop_words ='english')\n",
    "\n",
    "# list of models we would like to complete \n",
    "models = [\n",
    "#LogisticRegressionCV(cv=5,solver='saga',scoring='accuracy', random_state=1, n_jobs=-1),\n",
    "LinearRegression(),\n",
    "linear_model.Ridge(alpha=.5),\n",
    "linear_model.Lasso(alpha=0.1),\n",
    "#linear_model.LassoLars(alpha=.1, normalize=False),\n",
    "#linear_model.BayesianRidge()\n",
    "]\n",
    "\n",
    "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)\n",
    "print(y_test)\n",
    "print(q9_df['answer'].shape == q9_df['score_100'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "17b1c519",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearRegression Scored: 0.523\n",
      "\n",
      "Ridge Scored: 0.504\n",
      "\n",
      "Lasso Scored: 0.455\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train, test and store each model \n",
    "for model in models:\n",
    "\n",
    "    # our pipeline is changed to accept model\n",
    "    clf = Pipeline(steps=[\n",
    "        ('features', TfidfVectorizer(**tfidf_params)),\n",
    "        ('model', model) #just model not model() as we have done that in models list\n",
    "    ])\n",
    "\n",
    "    clf.fit(X_train,y_train)\n",
    "    score = clf.score(X_test,y_test)\n",
    "\n",
    "    model_name = clf.named_steps['model'].__class__.__name__ # hack to get name\n",
    "\n",
    "    model_params = clf.named_steps['model']. get_params()\n",
    "\n",
    "\n",
    "    print(f'{model_name} Scored: {score:.3f}\\n')\n",
    "\n",
    "    bench_mark['model_name'].append(model_name)\n",
    "    bench_mark['score'].append(score)\n",
    "    bench_mark['model'].append(clf)\n",
    "    bench_mark['used_params'].append(model_params)\n",
    "\n",
    "# in the end, place the bench_mark to DataFrame\n",
    "models_df = pd.DataFrame(bench_mark)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b9eb454e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 44, 0, 25, 42, 50, 0, 56, 67, 11, 78, 67, 83, 0, 0, 67, 78, 56, 0]\n"
     ]
    }
   ],
   "source": [
    "print(list(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f20f981c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[21.16137995 24.69506275 29.52815267 28.3441583  36.13718027 39.99193895\n",
      " 31.69347105 17.26140398 47.9817318  58.36513803 13.15027654 82.40459233\n",
      " 17.68983516 80.13362958 25.52967382 54.07244353 77.77340838 74.49519663\n",
      " 47.43099836 10.12194831]\n"
     ]
    }
   ],
   "source": [
    "# now you have the trained modes in DataFrame, their scores and parameters. \n",
    "#You can access and use any model.\n",
    "\n",
    "linear_reg = models_df[models_df['model_name']=='LinearRegression']['model'].iloc[0]\n",
    "\n",
    "y_preds = linear_reg.predict(X_test)\n",
    "\n",
    "print(y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "91cdce11",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25.94227429 29.28723925 33.71166646 31.05884632 40.00118157 41.81467556\n",
      " 34.07090042 24.47092511 48.70075656 56.39986064 18.19676814 72.75633744\n",
      " 28.74305197 69.9408796  29.94776635 52.05950618 69.53626136 66.28512588\n",
      " 45.12586258 16.44291978]\n"
     ]
    }
   ],
   "source": [
    "linear_reg = models_df[models_df['model_name']=='Ridge']['model'].iloc[0]\n",
    "\n",
    "y_preds = linear_reg.predict(X_test)\n",
    "print(y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2fc04018",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11.56659477 22.64229647 42.43509219 44.13839448 57.92730671 51.95386457\n",
      " 38.63437517 37.94497399 61.06622356 44.17965749 22.85366507 82.17779014\n",
      " 27.24576589 64.92348456 20.15174529 17.95243658 87.64820445 72.30721986\n",
      " 48.10113282 37.94497399]\n"
     ]
    }
   ],
   "source": [
    "linear_reg = models_df[models_df['model_name']=='Lasso']['model'].iloc[0]\n",
    "\n",
    "y_preds = linear_reg.predict(X_test)\n",
    "print(y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d55bcaa",
   "metadata": {},
   "source": [
    "# Text Classification using BERT and TensorFlow\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "942ad9be",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(q4_df['answer'], q4_df['score_4'], stratify=q4_df['score_4'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2af1b43d",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow_text'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Input \u001b[0;32mIn [30]\u001b[0m, in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow_hub\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mhub\u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;66;03m#!pip install tensorflow-text\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow_text\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mtext\u001b[39;00m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow_text'"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "#!pip install tensorflow-text\n",
    "import tensorflow_text as text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e3402477",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU')]\n"
     ]
    }
   ],
   "source": [
    "print(tf.config.list_physical_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c47b0a0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}