Features Engineering and Data Prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "92660e4d",
   "metadata": {},
   "source": [
    "# Features Engineering "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cccb12db",
   "metadata": {},
   "source": [
    "### Engineering Features\n",
    "    :: Features Selection\n",
    "    :: Features Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "41fbcdfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "37d2657d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data_cleaned.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "db877c6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_year</th>\n",
       "      <th>experience_level</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>job_title</th>\n",
       "      <th>salary_currency</th>\n",
       "      <th>salary_in_usd</th>\n",
       "      <th>employee_residence</th>\n",
       "      <th>remote_ratio</th>\n",
       "      <th>company_location</th>\n",
       "      <th>company_size</th>\n",
       "      <th>senior</th>\n",
       "      <th>roles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020</td>\n",
       "      <td>MI</td>\n",
       "      <td>FT</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>EUR</td>\n",
       "      <td>79833</td>\n",
       "      <td>DE</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DE</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>DS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>Machine Learning Scientist</td>\n",
       "      <td>USD</td>\n",
       "      <td>260000</td>\n",
       "      <td>JP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>JP</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "      <td>ML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>Big Data Engineer</td>\n",
       "      <td>GBP</td>\n",
       "      <td>109024</td>\n",
       "      <td>GB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>GB</td>\n",
       "      <td>M</td>\n",
       "      <td>0</td>\n",
       "      <td>DE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020</td>\n",
       "      <td>MI</td>\n",
       "      <td>FT</td>\n",
       "      <td>Product Data Analyst</td>\n",
       "      <td>USD</td>\n",
       "      <td>20000</td>\n",
       "      <td>HN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HN</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "      <td>AN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>Machine Learning Engineer</td>\n",
       "      <td>USD</td>\n",
       "      <td>150000</td>\n",
       "      <td>US</td>\n",
       "      <td>0.5</td>\n",
       "      <td>US</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>ML</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   work_year experience_level employment_type                   job_title  \\\n",
       "0       2020               MI              FT              Data Scientist   \n",
       "1       2020               SE              FT  Machine Learning Scientist   \n",
       "2       2020               SE              FT           Big Data Engineer   \n",
       "3       2020               MI              FT        Product Data Analyst   \n",
       "4       2020               SE              FT   Machine Learning Engineer   \n",
       "\n",
       "  salary_currency  salary_in_usd employee_residence  remote_ratio  \\\n",
       "0             EUR          79833                 DE           0.0   \n",
       "1             USD         260000                 JP           0.0   \n",
       "2             GBP         109024                 GB           0.5   \n",
       "3             USD          20000                 HN           0.0   \n",
       "4             USD         150000                 US           0.5   \n",
       "\n",
       "  company_location company_size  senior roles  \n",
       "0               DE            L       0    DS  \n",
       "1               JP            S       0    ML  \n",
       "2               GB            M       0    DE  \n",
       "3               HN            S       0    AN  \n",
       "4               US            L       0    ML  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4a9651a",
   "metadata": {},
   "source": [
    "#### Features Selection\n",
    "   From Exploration, Features such as roles, seniority, company size, remote ratio, employment type and experience level are most importance in predicting the Data Scientist's salary while the work year and employee residence are also key.\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b887ec83",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(\"job_title\",axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "223a1f65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['work_year', 'experience_level', 'employment_type', 'salary_in_usd',\n",
       "       'employee_residence', 'remote_ratio', 'company_location',\n",
       "       'company_size', 'senior', 'roles'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop(\"salary_currency\",axis=1,inplace = True)\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cadf67f",
   "metadata": {},
   "source": [
    "##### Features Encoding \n",
    "All of the features are Categorical "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a85ac254",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_year</th>\n",
       "      <th>experience_level</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>salary_in_usd</th>\n",
       "      <th>employee_residence</th>\n",
       "      <th>remote_ratio</th>\n",
       "      <th>company_location</th>\n",
       "      <th>company_size</th>\n",
       "      <th>senior</th>\n",
       "      <th>roles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020</td>\n",
       "      <td>MI</td>\n",
       "      <td>FT</td>\n",
       "      <td>79833</td>\n",
       "      <td>DE</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DE</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>DS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>260000</td>\n",
       "      <td>JP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>JP</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "      <td>ML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>109024</td>\n",
       "      <td>GB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>GB</td>\n",
       "      <td>M</td>\n",
       "      <td>0</td>\n",
       "      <td>DE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020</td>\n",
       "      <td>MI</td>\n",
       "      <td>FT</td>\n",
       "      <td>20000</td>\n",
       "      <td>HN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HN</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "      <td>AN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>150000</td>\n",
       "      <td>US</td>\n",
       "      <td>0.5</td>\n",
       "      <td>US</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>ML</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   work_year experience_level employment_type  salary_in_usd  \\\n",
       "0       2020               MI              FT          79833   \n",
       "1       2020               SE              FT         260000   \n",
       "2       2020               SE              FT         109024   \n",
       "3       2020               MI              FT          20000   \n",
       "4       2020               SE              FT         150000   \n",
       "\n",
       "  employee_residence  remote_ratio company_location company_size  senior roles  \n",
       "0                 DE           0.0               DE            L       0    DS  \n",
       "1                 JP           0.0               JP            S       0    ML  \n",
       "2                 GB           0.5               GB            M       0    DE  \n",
       "3                 HN           0.0               HN            S       0    AN  \n",
       "4                 US           0.5               US            L       0    ML  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "50e39101",
   "metadata": {},
   "outputs": [],
   "source": [
    "dum = pd.get_dummies(df[[\"employment_type\",\"roles\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a4e5b987",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df,dum],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7d9444a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_year</th>\n",
       "      <th>experience_level</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>salary_in_usd</th>\n",
       "      <th>employee_residence</th>\n",
       "      <th>remote_ratio</th>\n",
       "      <th>company_location</th>\n",
       "      <th>company_size</th>\n",
       "      <th>senior</th>\n",
       "      <th>roles</th>\n",
       "      <th>...</th>\n",
       "      <th>employment_type_FT</th>\n",
       "      <th>employment_type_PT</th>\n",
       "      <th>roles_AI</th>\n",
       "      <th>roles_AN</th>\n",
       "      <th>roles_CV</th>\n",
       "      <th>roles_DE</th>\n",
       "      <th>roles_DS</th>\n",
       "      <th>roles_ML</th>\n",
       "      <th>roles_OT</th>\n",
       "      <th>roles_RE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020</td>\n",
       "      <td>MI</td>\n",
       "      <td>FT</td>\n",
       "      <td>79833</td>\n",
       "      <td>DE</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DE</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>DS</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>260000</td>\n",
       "      <td>JP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>JP</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "      <td>ML</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>109024</td>\n",
       "      <td>GB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>GB</td>\n",
       "      <td>M</td>\n",
       "      <td>0</td>\n",
       "      <td>DE</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020</td>\n",
       "      <td>MI</td>\n",
       "      <td>FT</td>\n",
       "      <td>20000</td>\n",
       "      <td>HN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HN</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "      <td>AN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>150000</td>\n",
       "      <td>US</td>\n",
       "      <td>0.5</td>\n",
       "      <td>US</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>ML</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>602</th>\n",
       "      <td>2022</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>154000</td>\n",
       "      <td>US</td>\n",
       "      <td>1.0</td>\n",
       "      <td>US</td>\n",
       "      <td>M</td>\n",
       "      <td>0</td>\n",
       "      <td>DE</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>603</th>\n",
       "      <td>2022</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>126000</td>\n",
       "      <td>US</td>\n",
       "      <td>1.0</td>\n",
       "      <td>US</td>\n",
       "      <td>M</td>\n",
       "      <td>0</td>\n",
       "      <td>DE</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>2022</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>129000</td>\n",
       "      <td>US</td>\n",
       "      <td>0.0</td>\n",
       "      <td>US</td>\n",
       "      <td>M</td>\n",
       "      <td>0</td>\n",
       "      <td>AN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>2022</td>\n",
       "      <td>SE</td>\n",
       "      <td>FT</td>\n",
       "      <td>150000</td>\n",
       "      <td>US</td>\n",
       "      <td>1.0</td>\n",
       "      <td>US</td>\n",
       "      <td>M</td>\n",
       "      <td>0</td>\n",
       "      <td>AN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>606</th>\n",
       "      <td>2022</td>\n",
       "      <td>MI</td>\n",
       "      <td>FT</td>\n",
       "      <td>200000</td>\n",
       "      <td>IN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>US</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>AI</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>607 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     work_year experience_level employment_type  salary_in_usd  \\\n",
       "0         2020               MI              FT          79833   \n",
       "1         2020               SE              FT         260000   \n",
       "2         2020               SE              FT         109024   \n",
       "3         2020               MI              FT          20000   \n",
       "4         2020               SE              FT         150000   \n",
       "..         ...              ...             ...            ...   \n",
       "602       2022               SE              FT         154000   \n",
       "603       2022               SE              FT         126000   \n",
       "604       2022               SE              FT         129000   \n",
       "605       2022               SE              FT         150000   \n",
       "606       2022               MI              FT         200000   \n",
       "\n",
       "    employee_residence  remote_ratio company_location company_size  senior  \\\n",
       "0                   DE           0.0               DE            L       0   \n",
       "1                   JP           0.0               JP            S       0   \n",
       "2                   GB           0.5               GB            M       0   \n",
       "3                   HN           0.0               HN            S       0   \n",
       "4                   US           0.5               US            L       0   \n",
       "..                 ...           ...              ...          ...     ...   \n",
       "602                 US           1.0               US            M       0   \n",
       "603                 US           1.0               US            M       0   \n",
       "604                 US           0.0               US            M       0   \n",
       "605                 US           1.0               US            M       0   \n",
       "606                 IN           1.0               US            L       0   \n",
       "\n",
       "    roles  ...  employment_type_FT  employment_type_PT  roles_AI  roles_AN  \\\n",
       "0      DS  ...                   1                   0         0         0   \n",
       "1      ML  ...                   1                   0         0         0   \n",
       "2      DE  ...                   1                   0         0         0   \n",
       "3      AN  ...                   1                   0         0         1   \n",
       "4      ML  ...                   1                   0         0         0   \n",
       "..    ...  ...                 ...                 ...       ...       ...   \n",
       "602    DE  ...                   1                   0         0         0   \n",
       "603    DE  ...                   1                   0         0         0   \n",
       "604    AN  ...                   1                   0         0         1   \n",
       "605    AN  ...                   1                   0         0         1   \n",
       "606    AI  ...                   1                   0         1         0   \n",
       "\n",
       "     roles_CV  roles_DE  roles_DS  roles_ML  roles_OT  roles_RE  \n",
       "0           0         0         1         0         0         0  \n",
       "1           0         0         0         1         0         0  \n",
       "2           0         1         0         0         0         0  \n",
       "3           0         0         0         0         0         0  \n",
       "4           0         0         0         1         0         0  \n",
       "..        ...       ...       ...       ...       ...       ...  \n",
       "602         0         1         0         0         0         0  \n",
       "603         0         1         0         0         0         0  \n",
       "604         0         0         0         0         0         0  \n",
       "605         0         0         0         0         0         0  \n",
       "606         0         0         0         0         0         0  \n",
       "\n",
       "[607 rows x 22 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "86af4a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop([\"employment_type\",\"roles\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d582bf5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.work_year = df.work_year-2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6840e483",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 =df\n",
    "df1.experience_level = df[\"experience_level\"].apply(lambda x: 4 if x == \"EX\" else 3 if x == \"SE\" else 2 if x == \"MI\" else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1bdbb652",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_year</th>\n",
       "      <th>experience_level</th>\n",
       "      <th>salary_in_usd</th>\n",
       "      <th>employee_residence</th>\n",
       "      <th>remote_ratio</th>\n",
       "      <th>company_location</th>\n",
       "      <th>company_size</th>\n",
       "      <th>senior</th>\n",
       "      <th>employment_type_CT</th>\n",
       "      <th>employment_type_FL</th>\n",
       "      <th>employment_type_FT</th>\n",
       "      <th>employment_type_PT</th>\n",
       "      <th>roles_AI</th>\n",
       "      <th>roles_AN</th>\n",
       "      <th>roles_CV</th>\n",
       "      <th>roles_DE</th>\n",
       "      <th>roles_DS</th>\n",
       "      <th>roles_ML</th>\n",
       "      <th>roles_OT</th>\n",
       "      <th>roles_RE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>79833</td>\n",
       "      <td>DE</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DE</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>260000</td>\n",
       "      <td>JP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>JP</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>109024</td>\n",
       "      <td>GB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>GB</td>\n",
       "      <td>M</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20000</td>\n",
       "      <td>HN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HN</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>150000</td>\n",
       "      <td>US</td>\n",
       "      <td>0.5</td>\n",
       "      <td>US</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   work_year  experience_level  salary_in_usd employee_residence  \\\n",
       "0          1                 2          79833                 DE   \n",
       "1          1                 3         260000                 JP   \n",
       "2          1                 3         109024                 GB   \n",
       "3          1                 2          20000                 HN   \n",
       "4          1                 3         150000                 US   \n",
       "\n",
       "   remote_ratio company_location company_size  senior  employment_type_CT  \\\n",
       "0           0.0               DE            L       0                   0   \n",
       "1           0.0               JP            S       0                   0   \n",
       "2           0.5               GB            M       0                   0   \n",
       "3           0.0               HN            S       0                   0   \n",
       "4           0.5               US            L       0                   0   \n",
       "\n",
       "   employment_type_FL  employment_type_FT  employment_type_PT  roles_AI  \\\n",
       "0                   0                   1                   0         0   \n",
       "1                   0                   1                   0         0   \n",
       "2                   0                   1                   0         0   \n",
       "3                   0                   1                   0         0   \n",
       "4                   0                   1                   0         0   \n",
       "\n",
       "   roles_AN  roles_CV  roles_DE  roles_DS  roles_ML  roles_OT  roles_RE  \n",
       "0         0         0         0         1         0         0         0  \n",
       "1         0         0         0         0         1         0         0  \n",
       "2         0         0         1         0         0         0         0  \n",
       "3         1         0         0         0         0         0         0  \n",
       "4         0         0         0         0         1         0         0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bf4fac29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['work_year', 'experience_level', 'salary_in_usd', 'employee_residence',\n",
       "       'remote_ratio', 'company_location', 'company_size', 'senior',\n",
       "       'employment_type_CT', 'employment_type_FL', 'employment_type_FT',\n",
       "       'employment_type_PT', 'roles_AI', 'roles_AN', 'roles_CV', 'roles_DE',\n",
       "       'roles_DS', 'roles_ML', 'roles_OT', 'roles_RE'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f6010cf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "23bd0218",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df.company_size = df[\"company_size\"].apply(lambda x: 3 if x == \"L\" else 2 if x == \"M\" else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b40bf4c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_year</th>\n",
       "      <th>experience_level</th>\n",
       "      <th>salary_in_usd</th>\n",
       "      <th>employee_residence</th>\n",
       "      <th>remote_ratio</th>\n",
       "      <th>company_location</th>\n",
       "      <th>company_size</th>\n",
       "      <th>senior</th>\n",
       "      <th>employment_type_CT</th>\n",
       "      <th>employment_type_FL</th>\n",
       "      <th>employment_type_FT</th>\n",
       "      <th>employment_type_PT</th>\n",
       "      <th>roles_AI</th>\n",
       "      <th>roles_AN</th>\n",
       "      <th>roles_CV</th>\n",
       "      <th>roles_DE</th>\n",
       "      <th>roles_DS</th>\n",
       "      <th>roles_ML</th>\n",
       "      <th>roles_OT</th>\n",
       "      <th>roles_RE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>79833</td>\n",
       "      <td>DE</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DE</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>260000</td>\n",
       "      <td>JP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>JP</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>109024</td>\n",
       "      <td>GB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>GB</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20000</td>\n",
       "      <td>HN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>150000</td>\n",
       "      <td>US</td>\n",
       "      <td>0.5</td>\n",
       "      <td>US</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   work_year  experience_level  salary_in_usd employee_residence  \\\n",
       "0          1                 2          79833                 DE   \n",
       "1          1                 3         260000                 JP   \n",
       "2          1                 3         109024                 GB   \n",
       "3          1                 2          20000                 HN   \n",
       "4          1                 3         150000                 US   \n",
       "\n",
       "   remote_ratio company_location  company_size  senior  employment_type_CT  \\\n",
       "0           0.0               DE             3       0                   0   \n",
       "1           0.0               JP             1       0                   0   \n",
       "2           0.5               GB             2       0                   0   \n",
       "3           0.0               HN             1       0                   0   \n",
       "4           0.5               US             3       0                   0   \n",
       "\n",
       "   employment_type_FL  employment_type_FT  employment_type_PT  roles_AI  \\\n",
       "0                   0                   1                   0         0   \n",
       "1                   0                   1                   0         0   \n",
       "2                   0                   1                   0         0   \n",
       "3                   0                   1                   0         0   \n",
       "4                   0                   1                   0         0   \n",
       "\n",
       "   roles_AN  roles_CV  roles_DE  roles_DS  roles_ML  roles_OT  roles_RE  \n",
       "0         0         0         0         1         0         0         0  \n",
       "1         0         0         0         0         1         0         0  \n",
       "2         0         0         1         0         0         0         0  \n",
       "3         1         0         0         0         0         0         0  \n",
       "4         0         0         0         0         1         0         0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9f78bc3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"away\"]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1d801905",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_year</th>\n",
       "      <th>experience_level</th>\n",
       "      <th>salary_in_usd</th>\n",
       "      <th>employee_residence</th>\n",
       "      <th>remote_ratio</th>\n",
       "      <th>company_location</th>\n",
       "      <th>company_size</th>\n",
       "      <th>senior</th>\n",
       "      <th>employment_type_CT</th>\n",
       "      <th>employment_type_FL</th>\n",
       "      <th>...</th>\n",
       "      <th>employment_type_PT</th>\n",
       "      <th>roles_AI</th>\n",
       "      <th>roles_AN</th>\n",
       "      <th>roles_CV</th>\n",
       "      <th>roles_DE</th>\n",
       "      <th>roles_DS</th>\n",
       "      <th>roles_ML</th>\n",
       "      <th>roles_OT</th>\n",
       "      <th>roles_RE</th>\n",
       "      <th>away</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>79833</td>\n",
       "      <td>DE</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DE</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>260000</td>\n",
       "      <td>JP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>JP</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>109024</td>\n",
       "      <td>GB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>GB</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20000</td>\n",
       "      <td>HN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>150000</td>\n",
       "      <td>US</td>\n",
       "      <td>0.5</td>\n",
       "      <td>US</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   work_year  experience_level  salary_in_usd employee_residence  \\\n",
       "0          1                 2          79833                 DE   \n",
       "1          1                 3         260000                 JP   \n",
       "2          1                 3         109024                 GB   \n",
       "3          1                 2          20000                 HN   \n",
       "4          1                 3         150000                 US   \n",
       "\n",
       "   remote_ratio company_location  company_size  senior  employment_type_CT  \\\n",
       "0           0.0               DE             3       0                   0   \n",
       "1           0.0               JP             1       0                   0   \n",
       "2           0.5               GB             2       0                   0   \n",
       "3           0.0               HN             1       0                   0   \n",
       "4           0.5               US             3       0                   0   \n",
       "\n",
       "   employment_type_FL  ...  employment_type_PT  roles_AI  roles_AN  roles_CV  \\\n",
       "0                   0  ...                   0         0         0         0   \n",
       "1                   0  ...                   0         0         0         0   \n",
       "2                   0  ...                   0         0         0         0   \n",
       "3                   0  ...                   0         0         1         0   \n",
       "4                   0  ...                   0         0         0         0   \n",
       "\n",
       "   roles_DE  roles_DS  roles_ML  roles_OT  roles_RE  away  \n",
       "0         0         1         0         0         0     0  \n",
       "1         0         0         1         0         0     0  \n",
       "2         1         0         0         0         0     0  \n",
       "3         0         0         0         0         0     0  \n",
       "4         0         0         1         0         0     0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3a20186c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Cypher\\AppData\\Local\\Temp/ipykernel_9112/2292137763.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df.away[df.employee_residence!=df.company_location] = 1\n"
     ]
    }
   ],
   "source": [
    "df.away[df.employee_residence!=df.company_location] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "025e8182",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    556\n",
       "1     51\n",
       "Name: away, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.away.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2b686aad",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop([\"employee_residence\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "68631de8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_year</th>\n",
       "      <th>experience_level</th>\n",
       "      <th>salary_in_usd</th>\n",
       "      <th>remote_ratio</th>\n",
       "      <th>company_location</th>\n",
       "      <th>company_size</th>\n",
       "      <th>senior</th>\n",
       "      <th>employment_type_CT</th>\n",
       "      <th>employment_type_FL</th>\n",
       "      <th>employment_type_FT</th>\n",
       "      <th>employment_type_PT</th>\n",
       "      <th>roles_AI</th>\n",
       "      <th>roles_AN</th>\n",
       "      <th>roles_CV</th>\n",
       "      <th>roles_DE</th>\n",
       "      <th>roles_DS</th>\n",
       "      <th>roles_ML</th>\n",
       "      <th>roles_OT</th>\n",
       "      <th>roles_RE</th>\n",
       "      <th>away</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>79833</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DE</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>260000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>JP</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>109024</td>\n",
       "      <td>0.5</td>\n",
       "      <td>GB</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>150000</td>\n",
       "      <td>0.5</td>\n",
       "      <td>US</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   work_year  experience_level  salary_in_usd  remote_ratio company_location  \\\n",
       "0          1                 2          79833           0.0               DE   \n",
       "1          1                 3         260000           0.0               JP   \n",
       "2          1                 3         109024           0.5               GB   \n",
       "3          1                 2          20000           0.0               HN   \n",
       "4          1                 3         150000           0.5               US   \n",
       "\n",
       "   company_size  senior  employment_type_CT  employment_type_FL  \\\n",
       "0             3       0                   0                   0   \n",
       "1             1       0                   0                   0   \n",
       "2             2       0                   0                   0   \n",
       "3             1       0                   0                   0   \n",
       "4             3       0                   0                   0   \n",
       "\n",
       "   employment_type_FT  employment_type_PT  roles_AI  roles_AN  roles_CV  \\\n",
       "0                   1                   0         0         0         0   \n",
       "1                   1                   0         0         0         0   \n",
       "2                   1                   0         0         0         0   \n",
       "3                   1                   0         0         1         0   \n",
       "4                   1                   0         0         0         0   \n",
       "\n",
       "   roles_DE  roles_DS  roles_ML  roles_OT  roles_RE  away  \n",
       "0         0         1         0         0         0     0  \n",
       "1         0         0         1         0         0     0  \n",
       "2         1         0         0         0         0     0  \n",
       "3         0         0         0         0         0     0  \n",
       "4         0         0         1         0         0     0  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0414ed96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "US    355\n",
       "GB     47\n",
       "CA     30\n",
       "DE     28\n",
       "IN     24\n",
       "FR     15\n",
       "ES     14\n",
       "GR     11\n",
       "JP      6\n",
       "NL      4\n",
       "AT      4\n",
       "PT      4\n",
       "PL      4\n",
       "LU      3\n",
       "PK      3\n",
       "BR      3\n",
       "AE      3\n",
       "MX      3\n",
       "AU      3\n",
       "TR      3\n",
       "DK      3\n",
       "IT      2\n",
       "CZ      2\n",
       "SI      2\n",
       "RU      2\n",
       "CH      2\n",
       "NG      2\n",
       "CN      2\n",
       "BE      2\n",
       "VN      1\n",
       "EE      1\n",
       "AS      1\n",
       "DZ      1\n",
       "MY      1\n",
       "MD      1\n",
       "KE      1\n",
       "SG      1\n",
       "CO      1\n",
       "IR      1\n",
       "CL      1\n",
       "MT      1\n",
       "IL      1\n",
       "UA      1\n",
       "IQ      1\n",
       "RO      1\n",
       "HR      1\n",
       "NZ      1\n",
       "HU      1\n",
       "HN      1\n",
       "IE      1\n",
       "Name: company_location, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.company_location.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "730d1ee0",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_frequency_map = df.company_location.value_counts().to_dict()\n",
    "df.company_location = df.company_location.map(X_frequency_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "96187c14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_year</th>\n",
       "      <th>experience_level</th>\n",
       "      <th>salary_in_usd</th>\n",
       "      <th>remote_ratio</th>\n",
       "      <th>company_location</th>\n",
       "      <th>company_size</th>\n",
       "      <th>senior</th>\n",
       "      <th>employment_type_CT</th>\n",
       "      <th>employment_type_FL</th>\n",
       "      <th>employment_type_FT</th>\n",
       "      <th>employment_type_PT</th>\n",
       "      <th>roles_AI</th>\n",
       "      <th>roles_AN</th>\n",
       "      <th>roles_CV</th>\n",
       "      <th>roles_DE</th>\n",
       "      <th>roles_DS</th>\n",
       "      <th>roles_ML</th>\n",
       "      <th>roles_OT</th>\n",
       "      <th>roles_RE</th>\n",
       "      <th>away</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>79833</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>260000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>109024</td>\n",
       "      <td>0.5</td>\n",
       "      <td>47</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>150000</td>\n",
       "      <td>0.5</td>\n",
       "      <td>355</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   work_year  experience_level  salary_in_usd  remote_ratio  company_location  \\\n",
       "0          1                 2          79833           0.0                28   \n",
       "1          1                 3         260000           0.0                 6   \n",
       "2          1                 3         109024           0.5                47   \n",
       "3          1                 2          20000           0.0                 1   \n",
       "4          1                 3         150000           0.5               355   \n",
       "\n",
       "   company_size  senior  employment_type_CT  employment_type_FL  \\\n",
       "0             3       0                   0                   0   \n",
       "1             1       0                   0                   0   \n",
       "2             2       0                   0                   0   \n",
       "3             1       0                   0                   0   \n",
       "4             3       0                   0                   0   \n",
       "\n",
       "   employment_type_FT  employment_type_PT  roles_AI  roles_AN  roles_CV  \\\n",
       "0                   1                   0         0         0         0   \n",
       "1                   1                   0         0         0         0   \n",
       "2                   1                   0         0         0         0   \n",
       "3                   1                   0         0         1         0   \n",
       "4                   1                   0         0         0         0   \n",
       "\n",
       "   roles_DE  roles_DS  roles_ML  roles_OT  roles_RE  away  \n",
       "0         0         1         0         0         0     0  \n",
       "1         0         0         1         0         0     0  \n",
       "2         1         0         0         0         0     0  \n",
       "3         0         0         0         0         0     0  \n",
       "4         0         0         1         0         0     0  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f81fc003",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(607, 20)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "d690ccc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"data_prepared.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec664a6f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b155de8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}