Team3_Data_Preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Team 3 - Final Project Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/secretstorage/dhcrypto.py:16: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead\n",
      "  from cryptography.utils import int_from_bytes\n",
      "/opt/conda/lib/python3.7/site-packages/secretstorage/util.py:25: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead\n",
      "  from cryptography.utils import int_from_bytes\n",
      "Collecting pyathena\n",
      "  Using cached PyAthena-2.5.1-py3-none-any.whl (40 kB)\n",
      "Requirement already satisfied: tenacity>=4.1.0 in /opt/conda/lib/python3.7/site-packages (from pyathena) (8.0.1)\n",
      "Requirement already satisfied: boto3>=1.4.4 in /opt/conda/lib/python3.7/site-packages (from pyathena) (1.20.23)\n",
      "Requirement already satisfied: botocore>=1.5.52 in /opt/conda/lib/python3.7/site-packages (from pyathena) (1.23.23)\n",
      "Requirement already satisfied: s3transfer<0.6.0,>=0.5.0 in /opt/conda/lib/python3.7/site-packages (from boto3>=1.4.4->pyathena) (0.5.0)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /opt/conda/lib/python3.7/site-packages (from boto3>=1.4.4->pyathena) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/conda/lib/python3.7/site-packages (from botocore>=1.5.52->pyathena) (2.8.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /opt/conda/lib/python3.7/site-packages (from botocore>=1.5.52->pyathena) (1.26.7)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil<3.0.0,>=2.1->botocore>=1.5.52->pyathena) (1.14.0)\n",
      "Installing collected packages: pyathena\n",
      "Successfully installed pyathena-2.5.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install pyathena"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "from botocore.exceptions import ClientError\n",
    "import sagemaker\n",
    "import pyathena as pa\n",
    "from pyathena.pandas.cursor import PandasCursor\n",
    "from datetime import datetime\n",
    "import os\n",
    "\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = sagemaker.Session()\n",
    "bucket = sess.default_bucket()\n",
    "role = sagemaker.get_execution_role()\n",
    "region = boto3.Session().region_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Athena modeling bucket\n",
    "s3_modeling_dir = 's3://ads508team3/modeling/'\n",
    "# Athena staging bucket\n",
    "s3_staging_dir = 's3://ads508team3/athena-staging/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Connection\n",
    "conn = pa.connect(region_name=region, s3_staging_dir=s3_staging_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dbname = 'myanimelist'\n",
    "table_list = ['animelist', 'anime', 'synopsis', 'ratings', 'watching_status']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Construct DataFrames\n",
    "\n",
    "**Bypassing the default pyathena cursor and using a more efficient PandasCursor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cursor = pa.connect(s3_staging_dir=s3_staging_dir, region_name='us-east-1',cursor_class=PandasCursor).cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "anime_list = cursor.execute('SELECT * FROM %s.%s' % (dbname, 'animelist')).as_pandas()\n",
    "anime = cursor.execute('SELECT * FROM %s.%s' % (dbname, 'anime')).as_pandas()\n",
    "#synopsis = cursor.execute('SELECT * FROM %s.%s' % (dbname, 'synopsis')).as_pandas()\n",
    "#ratings = cursor.execute('SELECT * FROM %s.%s' % (dbname, 'ratings')).as_pandas()\n",
    "watching_status = cursor.execute('SELECT * FROM %s.%s' % (dbname, 'watching_status')).as_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fix anime scores, replace 'Unknown' values with np.nan and then mean impute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "mal_id            Int64\n",
       "name             object\n",
       "score            object\n",
       "genres           object\n",
       "english_name     object\n",
       "japanese_name    object\n",
       "type             object\n",
       "episodes         object\n",
       "aired            object\n",
       "premiered        object\n",
       "producers        object\n",
       "licensors        object\n",
       "studios          object\n",
       "source           object\n",
       "duration         object\n",
       "rating           object\n",
       "ranked           object\n",
       "popularity        Int64\n",
       "members           Int64\n",
       "favorites         Int64\n",
       "watching          Int64\n",
       "completed         Int64\n",
       "on_hold           Int64\n",
       "dropped           Int64\n",
       "plan_to_watch    object\n",
       "score_10         object\n",
       "score_9          object\n",
       "score_8          object\n",
       "score_7          object\n",
       "score_6          object\n",
       "score_5          object\n",
       "score_4          object\n",
       "score_3          object\n",
       "score_2          object\n",
       "score_1          object\n",
       "dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace Unknown Values\n",
    "anime['score'] = anime['score'].replace({'Unknown': np.nan})\n",
    "anime['score'] = anime['score'].astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['score'].dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5141"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['score'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We may have to change this imputation method if it impacts the model too much\n",
    "anime['score'] = anime['score'].fillna(anime['score'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        8.780000\n",
       "1        8.390000\n",
       "2        8.240000\n",
       "3        7.270000\n",
       "4        6.980000\n",
       "           ...   \n",
       "17557    6.509999\n",
       "17558    6.509999\n",
       "17559    6.509999\n",
       "17560    6.509999\n",
       "17561    6.509999\n",
       "Name: score, Length: 17562, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['score']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Days Since Aired\n",
    "Pull the first date from Aired, convert to datetime and get timedelta in days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "anime['days_since_aired'] = anime['aired'].str.split(' to ', expand=True).iloc[:, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove spaces left and right\n",
    "anime['days_since_aired'] = anime['days_since_aired'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         Apr 3, 1998\n",
       "1         Sep 1, 2001\n",
       "2         Apr 1, 1998\n",
       "3         Jul 2, 2002\n",
       "4        Sep 30, 2004\n",
       "             ...     \n",
       "17557     Apr 4, 2021\n",
       "17558     Jan 1, 2021\n",
       "17559     Jul 1, 2021\n",
       "17560         Unknown\n",
       "17561     Jul 1, 2021\n",
       "Name: days_since_aired, Length: 17562, dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fix date format issues\n",
    "anime['days_since_aired'] = anime['days_since_aired'].str.replace('^\\w{3}, \\d{4}', lambda x: x[0][0:3] + ' 1, ' + x[0][5:9], regex=True)\n",
    "anime['days_since_aired'] = anime['days_since_aired'].str.replace('^\\d{4}', lambda x: 'Jan 1, ' + x[0], regex=True)\n",
    "anime['days_since_aired']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         Apr 3, 1998\n",
       "1         Sep 1, 2001\n",
       "2         Apr 1, 1998\n",
       "3         Jul 2, 2002\n",
       "4        Sep 30, 2004\n",
       "             ...     \n",
       "17557     Apr 4, 2021\n",
       "17558     Jan 1, 2021\n",
       "17559     Jul 1, 2021\n",
       "17560             NaN\n",
       "17561     Jul 1, 2021\n",
       "Name: days_since_aired, Length: 17562, dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['days_since_aired'] = anime['days_since_aired'].replace('Unknown', np.nan)\n",
    "anime['days_since_aired']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "anime['days_since_aired'] = pd.to_datetime(anime['days_since_aired'], format='%b %d, %Y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:1: FutureWarning: The pandas.datetime class is deprecated and will be removed from pandas in a future version. Import from datetime module instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "anime['days_since_aired'] = anime['days_since_aired'].fillna(pd.datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       1998-04-03 00:00:00.000000\n",
       "1       2001-09-01 00:00:00.000000\n",
       "2       1998-04-01 00:00:00.000000\n",
       "3       2002-07-02 00:00:00.000000\n",
       "4       2004-09-30 00:00:00.000000\n",
       "                   ...            \n",
       "17557   2021-04-04 00:00:00.000000\n",
       "17558   2021-01-01 00:00:00.000000\n",
       "17559   2021-07-01 00:00:00.000000\n",
       "17560   2022-04-02 23:27:00.193661\n",
       "17561   2021-07-01 00:00:00.000000\n",
       "Name: days_since_aired, Length: 17562, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['days_since_aired']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Time Delta in days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:1: FutureWarning: The pandas.datetime class is deprecated and will be removed from pandas in a future version. Import from datetime module instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "anime['days_since_aired'] = (pd.datetime.now() - anime['days_since_aired']).dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        8765\n",
       "1        7518\n",
       "2        8767\n",
       "3        7214\n",
       "4        6393\n",
       "         ... \n",
       "17557     363\n",
       "17558     456\n",
       "17559     275\n",
       "17560       0\n",
       "17561     275\n",
       "Name: days_since_aired, Length: 17562, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['days_since_aired']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One Hot Encode Genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_list = anime['genres'].apply(lambda x: x.split(', '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Sci-Fi',\n",
       " 'Drama',\n",
       " 'Military',\n",
       " 'Comedy',\n",
       " 'Mecha',\n",
       " 'Yuri',\n",
       " 'Vampire',\n",
       " 'Hentai',\n",
       " 'Super Power',\n",
       " 'Shounen Ai',\n",
       " 'Cars',\n",
       " 'Parody',\n",
       " 'Music',\n",
       " 'Psychological',\n",
       " 'Horror',\n",
       " 'Shoujo',\n",
       " 'Fantasy',\n",
       " 'Mystery',\n",
       " 'Ecchi',\n",
       " 'Magic',\n",
       " 'Harem',\n",
       " 'Adventure',\n",
       " 'Slice of Life',\n",
       " 'Kids',\n",
       " 'Josei',\n",
       " 'Supernatural',\n",
       " 'Shoujo Ai',\n",
       " 'Seinen',\n",
       " 'Samurai',\n",
       " 'Demons',\n",
       " 'Thriller',\n",
       " 'Unknown',\n",
       " 'Game',\n",
       " 'Dementia',\n",
       " 'Space',\n",
       " 'Shounen',\n",
       " 'Action',\n",
       " 'Martial Arts',\n",
       " 'Historical',\n",
       " 'Yaoi',\n",
       " 'Romance',\n",
       " 'Police',\n",
       " 'Sports',\n",
       " 'School']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Unique Genre\n",
    "list(set(genre_list.sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlb = MultiLabelBinarizer()\n",
    "ohe = pd.DataFrame(mlb.fit_transform(genre_list), columns=mlb.classes_, index=genre_list.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "anime = pd.concat([anime, ohe], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mal_id</th>\n",
       "      <th>name</th>\n",
       "      <th>score</th>\n",
       "      <th>genres</th>\n",
       "      <th>english_name</th>\n",
       "      <th>japanese_name</th>\n",
       "      <th>type</th>\n",
       "      <th>episodes</th>\n",
       "      <th>aired</th>\n",
       "      <th>premiered</th>\n",
       "      <th>...</th>\n",
       "      <th>Slice of Life</th>\n",
       "      <th>Space</th>\n",
       "      <th>Sports</th>\n",
       "      <th>Super Power</th>\n",
       "      <th>Supernatural</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>Unknown</th>\n",
       "      <th>Vampire</th>\n",
       "      <th>Yaoi</th>\n",
       "      <th>Yuri</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Cowboy Bebop</td>\n",
       "      <td>8.780000</td>\n",
       "      <td>Action, Adventure, Comedy, Drama, Sci-Fi, Space</td>\n",
       "      <td>Cowboy Bebop</td>\n",
       "      <td>カウボーイビバップ</td>\n",
       "      <td>TV</td>\n",
       "      <td>26</td>\n",
       "      <td>Apr 3, 1998 to Apr 24, 1999</td>\n",
       "      <td>Spring 1998</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>Cowboy Bebop: Tengoku no Tobira</td>\n",
       "      <td>8.390000</td>\n",
       "      <td>Action, Drama, Mystery, Sci-Fi, Space</td>\n",
       "      <td>Cowboy Bebop:The Movie</td>\n",
       "      <td>カウボーイビバップ 天国の扉</td>\n",
       "      <td>Movie</td>\n",
       "      <td>1</td>\n",
       "      <td>Sep 1, 2001</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>Trigun</td>\n",
       "      <td>8.240000</td>\n",
       "      <td>Action, Sci-Fi, Adventure, Comedy, Drama, Shounen</td>\n",
       "      <td>Trigun</td>\n",
       "      <td>トライガン</td>\n",
       "      <td>TV</td>\n",
       "      <td>26</td>\n",
       "      <td>Apr 1, 1998 to Sep 30, 1998</td>\n",
       "      <td>Spring 1998</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>Witch Hunter Robin</td>\n",
       "      <td>7.270000</td>\n",
       "      <td>Action, Mystery, Police, Supernatural, Drama, ...</td>\n",
       "      <td>Witch Hunter Robin</td>\n",
       "      <td>Witch Hunter ROBIN (ウイッチハンターロビン)</td>\n",
       "      <td>TV</td>\n",
       "      <td>26</td>\n",
       "      <td>Jul 2, 2002 to Dec 24, 2002</td>\n",
       "      <td>Summer 2002</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>Bouken Ou Beet</td>\n",
       "      <td>6.980000</td>\n",
       "      <td>Adventure, Fantasy, Shounen, Supernatural</td>\n",
       "      <td>Beet the Vandel Buster</td>\n",
       "      <td>冒険王ビィト</td>\n",
       "      <td>TV</td>\n",
       "      <td>52</td>\n",
       "      <td>Sep 30, 2004 to Sep 29, 2005</td>\n",
       "      <td>Fall 2004</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17557</th>\n",
       "      <td>48481</td>\n",
       "      <td>Daomu Biji Zhi Qinling Shen Shu</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Adventure, Mystery, Supernatural</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>盗墓笔记之秦岭神树</td>\n",
       "      <td>ONA</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Apr 4, 2021 to ?</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17558</th>\n",
       "      <td>48483</td>\n",
       "      <td>Mieruko-chan</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Comedy, Horror, Supernatural</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>見える子ちゃん</td>\n",
       "      <td>TV</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>2021 to ?</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17559</th>\n",
       "      <td>48488</td>\n",
       "      <td>Higurashi no Naku Koro ni Sotsu</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Mystery, Dementia, Horror, Psychological, Supe...</td>\n",
       "      <td>Higurashi:When They Cry – SOTSU</td>\n",
       "      <td>ひぐらしのなく頃に卒</td>\n",
       "      <td>TV</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Jul, 2021 to ?</td>\n",
       "      <td>Summer 2021</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17560</th>\n",
       "      <td>48491</td>\n",
       "      <td>Yama no Susume: Next Summit</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Adventure, Slice of Life, Comedy</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>ヤマノススメ Next Summit</td>\n",
       "      <td>TV</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17561</th>\n",
       "      <td>48492</td>\n",
       "      <td>Scarlet Nexus</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Action, Fantasy</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>SCARLET NEXUS</td>\n",
       "      <td>TV</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Jul, 2021 to ?</td>\n",
       "      <td>Summer 2021</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17562 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       mal_id                             name     score  \\\n",
       "0           1                     Cowboy Bebop  8.780000   \n",
       "1           5  Cowboy Bebop: Tengoku no Tobira  8.390000   \n",
       "2           6                           Trigun  8.240000   \n",
       "3           7               Witch Hunter Robin  7.270000   \n",
       "4           8                   Bouken Ou Beet  6.980000   \n",
       "...       ...                              ...       ...   \n",
       "17557   48481  Daomu Biji Zhi Qinling Shen Shu  6.509999   \n",
       "17558   48483                     Mieruko-chan  6.509999   \n",
       "17559   48488  Higurashi no Naku Koro ni Sotsu  6.509999   \n",
       "17560   48491      Yama no Susume: Next Summit  6.509999   \n",
       "17561   48492                    Scarlet Nexus  6.509999   \n",
       "\n",
       "                                                  genres  \\\n",
       "0        Action, Adventure, Comedy, Drama, Sci-Fi, Space   \n",
       "1                  Action, Drama, Mystery, Sci-Fi, Space   \n",
       "2      Action, Sci-Fi, Adventure, Comedy, Drama, Shounen   \n",
       "3      Action, Mystery, Police, Supernatural, Drama, ...   \n",
       "4              Adventure, Fantasy, Shounen, Supernatural   \n",
       "...                                                  ...   \n",
       "17557                   Adventure, Mystery, Supernatural   \n",
       "17558                       Comedy, Horror, Supernatural   \n",
       "17559  Mystery, Dementia, Horror, Psychological, Supe...   \n",
       "17560                   Adventure, Slice of Life, Comedy   \n",
       "17561                                    Action, Fantasy   \n",
       "\n",
       "                          english_name                     japanese_name  \\\n",
       "0                         Cowboy Bebop                         カウボーイビバップ   \n",
       "1               Cowboy Bebop:The Movie                    カウボーイビバップ 天国の扉   \n",
       "2                               Trigun                             トライガン   \n",
       "3                   Witch Hunter Robin  Witch Hunter ROBIN (ウイッチハンターロビン)   \n",
       "4               Beet the Vandel Buster                            冒険王ビィト   \n",
       "...                                ...                               ...   \n",
       "17557                          Unknown                         盗墓笔记之秦岭神树   \n",
       "17558                          Unknown                           見える子ちゃん   \n",
       "17559  Higurashi:When They Cry – SOTSU                        ひぐらしのなく頃に卒   \n",
       "17560                          Unknown                ヤマノススメ Next Summit   \n",
       "17561                          Unknown                     SCARLET NEXUS   \n",
       "\n",
       "        type episodes                         aired    premiered  ...  \\\n",
       "0         TV       26   Apr 3, 1998 to Apr 24, 1999  Spring 1998  ...   \n",
       "1      Movie        1                   Sep 1, 2001      Unknown  ...   \n",
       "2         TV       26   Apr 1, 1998 to Sep 30, 1998  Spring 1998  ...   \n",
       "3         TV       26   Jul 2, 2002 to Dec 24, 2002  Summer 2002  ...   \n",
       "4         TV       52  Sep 30, 2004 to Sep 29, 2005    Fall 2004  ...   \n",
       "...      ...      ...                           ...          ...  ...   \n",
       "17557    ONA  Unknown              Apr 4, 2021 to ?      Unknown  ...   \n",
       "17558     TV  Unknown                     2021 to ?      Unknown  ...   \n",
       "17559     TV  Unknown                Jul, 2021 to ?  Summer 2021  ...   \n",
       "17560     TV  Unknown                       Unknown      Unknown  ...   \n",
       "17561     TV  Unknown                Jul, 2021 to ?  Summer 2021  ...   \n",
       "\n",
       "      Slice of Life Space Sports Super Power Supernatural Thriller Unknown  \\\n",
       "0                 0     1      0           0            0        0       0   \n",
       "1                 0     1      0           0            0        0       0   \n",
       "2                 0     0      0           0            0        0       0   \n",
       "3                 0     0      0           0            1        0       0   \n",
       "4                 0     0      0           0            1        0       0   \n",
       "...             ...   ...    ...         ...          ...      ...     ...   \n",
       "17557             0     0      0           0            1        0       0   \n",
       "17558             0     0      0           0            1        0       0   \n",
       "17559             0     0      0           0            1        1       0   \n",
       "17560             1     0      0           0            0        0       0   \n",
       "17561             0     0      0           0            0        0       0   \n",
       "\n",
       "       Vampire  Yaoi  Yuri  \n",
       "0            0     0     0  \n",
       "1            0     0     0  \n",
       "2            0     0     0  \n",
       "3            0     0     0  \n",
       "4            0     0     0  \n",
       "...        ...   ...   ...  \n",
       "17557        0     0     0  \n",
       "17558        0     0     0  \n",
       "17559        0     0     0  \n",
       "17560        0     0     0  \n",
       "17561        0     0     0  \n",
       "\n",
       "[17562 rows x 80 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One hot encode Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['TV', 'Movie', 'OVA', 'Special', 'ONA', 'Music', 'Unknown'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "anime['type'] = anime['type'].replace({'Unknown': 'Other'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc = OneHotEncoder(handle_unknown='ignore', sparse=False)\n",
    "type_ohe = pd.DataFrame(enc.fit_transform(anime[['type']]), columns=list(enc.categories_[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie</th>\n",
       "      <th>Music</th>\n",
       "      <th>ONA</th>\n",
       "      <th>OVA</th>\n",
       "      <th>Other</th>\n",
       "      <th>Special</th>\n",
       "      <th>TV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17557</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17558</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17559</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17560</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17561</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17562 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Movie  Music  ONA  OVA  Other  Special   TV\n",
       "0        0.0    0.0  0.0  0.0    0.0      0.0  1.0\n",
       "1        1.0    0.0  0.0  0.0    0.0      0.0  0.0\n",
       "2        0.0    0.0  0.0  0.0    0.0      0.0  1.0\n",
       "3        0.0    0.0  0.0  0.0    0.0      0.0  1.0\n",
       "4        0.0    0.0  0.0  0.0    0.0      0.0  1.0\n",
       "...      ...    ...  ...  ...    ...      ...  ...\n",
       "17557    0.0    0.0  1.0  0.0    0.0      0.0  0.0\n",
       "17558    0.0    0.0  0.0  0.0    0.0      0.0  1.0\n",
       "17559    0.0    0.0  0.0  0.0    0.0      0.0  1.0\n",
       "17560    0.0    0.0  0.0  0.0    0.0      0.0  1.0\n",
       "17561    0.0    0.0  0.0  0.0    0.0      0.0  1.0\n",
       "\n",
       "[17562 rows x 7 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type_ohe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concat\n",
    "anime = pd.concat([anime, type_ohe], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mal_id</th>\n",
       "      <th>name</th>\n",
       "      <th>score</th>\n",
       "      <th>genres</th>\n",
       "      <th>english_name</th>\n",
       "      <th>japanese_name</th>\n",
       "      <th>type</th>\n",
       "      <th>episodes</th>\n",
       "      <th>aired</th>\n",
       "      <th>premiered</th>\n",
       "      <th>...</th>\n",
       "      <th>Vampire</th>\n",
       "      <th>Yaoi</th>\n",
       "      <th>Yuri</th>\n",
       "      <th>Movie</th>\n",
       "      <th>Music</th>\n",
       "      <th>ONA</th>\n",
       "      <th>OVA</th>\n",
       "      <th>Other</th>\n",
       "      <th>Special</th>\n",
       "      <th>TV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Cowboy Bebop</td>\n",
       "      <td>8.780000</td>\n",
       "      <td>Action, Adventure, Comedy, Drama, Sci-Fi, Space</td>\n",
       "      <td>Cowboy Bebop</td>\n",
       "      <td>カウボーイビバップ</td>\n",
       "      <td>TV</td>\n",
       "      <td>26</td>\n",
       "      <td>Apr 3, 1998 to Apr 24, 1999</td>\n",
       "      <td>Spring 1998</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>Cowboy Bebop: Tengoku no Tobira</td>\n",
       "      <td>8.390000</td>\n",
       "      <td>Action, Drama, Mystery, Sci-Fi, Space</td>\n",
       "      <td>Cowboy Bebop:The Movie</td>\n",
       "      <td>カウボーイビバップ 天国の扉</td>\n",
       "      <td>Movie</td>\n",
       "      <td>1</td>\n",
       "      <td>Sep 1, 2001</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>Trigun</td>\n",
       "      <td>8.240000</td>\n",
       "      <td>Action, Sci-Fi, Adventure, Comedy, Drama, Shounen</td>\n",
       "      <td>Trigun</td>\n",
       "      <td>トライガン</td>\n",
       "      <td>TV</td>\n",
       "      <td>26</td>\n",
       "      <td>Apr 1, 1998 to Sep 30, 1998</td>\n",
       "      <td>Spring 1998</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>Witch Hunter Robin</td>\n",
       "      <td>7.270000</td>\n",
       "      <td>Action, Mystery, Police, Supernatural, Drama, ...</td>\n",
       "      <td>Witch Hunter Robin</td>\n",
       "      <td>Witch Hunter ROBIN (ウイッチハンターロビン)</td>\n",
       "      <td>TV</td>\n",
       "      <td>26</td>\n",
       "      <td>Jul 2, 2002 to Dec 24, 2002</td>\n",
       "      <td>Summer 2002</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>Bouken Ou Beet</td>\n",
       "      <td>6.980000</td>\n",
       "      <td>Adventure, Fantasy, Shounen, Supernatural</td>\n",
       "      <td>Beet the Vandel Buster</td>\n",
       "      <td>冒険王ビィト</td>\n",
       "      <td>TV</td>\n",
       "      <td>52</td>\n",
       "      <td>Sep 30, 2004 to Sep 29, 2005</td>\n",
       "      <td>Fall 2004</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17557</th>\n",
       "      <td>48481</td>\n",
       "      <td>Daomu Biji Zhi Qinling Shen Shu</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Adventure, Mystery, Supernatural</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>盗墓笔记之秦岭神树</td>\n",
       "      <td>ONA</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Apr 4, 2021 to ?</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17558</th>\n",
       "      <td>48483</td>\n",
       "      <td>Mieruko-chan</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Comedy, Horror, Supernatural</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>見える子ちゃん</td>\n",
       "      <td>TV</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>2021 to ?</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17559</th>\n",
       "      <td>48488</td>\n",
       "      <td>Higurashi no Naku Koro ni Sotsu</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Mystery, Dementia, Horror, Psychological, Supe...</td>\n",
       "      <td>Higurashi:When They Cry – SOTSU</td>\n",
       "      <td>ひぐらしのなく頃に卒</td>\n",
       "      <td>TV</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Jul, 2021 to ?</td>\n",
       "      <td>Summer 2021</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17560</th>\n",
       "      <td>48491</td>\n",
       "      <td>Yama no Susume: Next Summit</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Adventure, Slice of Life, Comedy</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>ヤマノススメ Next Summit</td>\n",
       "      <td>TV</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17561</th>\n",
       "      <td>48492</td>\n",
       "      <td>Scarlet Nexus</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>Action, Fantasy</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>SCARLET NEXUS</td>\n",
       "      <td>TV</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Jul, 2021 to ?</td>\n",
       "      <td>Summer 2021</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17562 rows × 87 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       mal_id                             name     score  \\\n",
       "0           1                     Cowboy Bebop  8.780000   \n",
       "1           5  Cowboy Bebop: Tengoku no Tobira  8.390000   \n",
       "2           6                           Trigun  8.240000   \n",
       "3           7               Witch Hunter Robin  7.270000   \n",
       "4           8                   Bouken Ou Beet  6.980000   \n",
       "...       ...                              ...       ...   \n",
       "17557   48481  Daomu Biji Zhi Qinling Shen Shu  6.509999   \n",
       "17558   48483                     Mieruko-chan  6.509999   \n",
       "17559   48488  Higurashi no Naku Koro ni Sotsu  6.509999   \n",
       "17560   48491      Yama no Susume: Next Summit  6.509999   \n",
       "17561   48492                    Scarlet Nexus  6.509999   \n",
       "\n",
       "                                                  genres  \\\n",
       "0        Action, Adventure, Comedy, Drama, Sci-Fi, Space   \n",
       "1                  Action, Drama, Mystery, Sci-Fi, Space   \n",
       "2      Action, Sci-Fi, Adventure, Comedy, Drama, Shounen   \n",
       "3      Action, Mystery, Police, Supernatural, Drama, ...   \n",
       "4              Adventure, Fantasy, Shounen, Supernatural   \n",
       "...                                                  ...   \n",
       "17557                   Adventure, Mystery, Supernatural   \n",
       "17558                       Comedy, Horror, Supernatural   \n",
       "17559  Mystery, Dementia, Horror, Psychological, Supe...   \n",
       "17560                   Adventure, Slice of Life, Comedy   \n",
       "17561                                    Action, Fantasy   \n",
       "\n",
       "                          english_name                     japanese_name  \\\n",
       "0                         Cowboy Bebop                         カウボーイビバップ   \n",
       "1               Cowboy Bebop:The Movie                    カウボーイビバップ 天国の扉   \n",
       "2                               Trigun                             トライガン   \n",
       "3                   Witch Hunter Robin  Witch Hunter ROBIN (ウイッチハンターロビン)   \n",
       "4               Beet the Vandel Buster                            冒険王ビィト   \n",
       "...                                ...                               ...   \n",
       "17557                          Unknown                         盗墓笔记之秦岭神树   \n",
       "17558                          Unknown                           見える子ちゃん   \n",
       "17559  Higurashi:When They Cry – SOTSU                        ひぐらしのなく頃に卒   \n",
       "17560                          Unknown                ヤマノススメ Next Summit   \n",
       "17561                          Unknown                     SCARLET NEXUS   \n",
       "\n",
       "        type episodes                         aired    premiered  ... Vampire  \\\n",
       "0         TV       26   Apr 3, 1998 to Apr 24, 1999  Spring 1998  ...       0   \n",
       "1      Movie        1                   Sep 1, 2001      Unknown  ...       0   \n",
       "2         TV       26   Apr 1, 1998 to Sep 30, 1998  Spring 1998  ...       0   \n",
       "3         TV       26   Jul 2, 2002 to Dec 24, 2002  Summer 2002  ...       0   \n",
       "4         TV       52  Sep 30, 2004 to Sep 29, 2005    Fall 2004  ...       0   \n",
       "...      ...      ...                           ...          ...  ...     ...   \n",
       "17557    ONA  Unknown              Apr 4, 2021 to ?      Unknown  ...       0   \n",
       "17558     TV  Unknown                     2021 to ?      Unknown  ...       0   \n",
       "17559     TV  Unknown                Jul, 2021 to ?  Summer 2021  ...       0   \n",
       "17560     TV  Unknown                       Unknown      Unknown  ...       0   \n",
       "17561     TV  Unknown                Jul, 2021 to ?  Summer 2021  ...       0   \n",
       "\n",
       "      Yaoi Yuri Movie Music  ONA  OVA  Other  Special   TV  \n",
       "0        0    0   0.0   0.0  0.0  0.0    0.0      0.0  1.0  \n",
       "1        0    0   1.0   0.0  0.0  0.0    0.0      0.0  0.0  \n",
       "2        0    0   0.0   0.0  0.0  0.0    0.0      0.0  1.0  \n",
       "3        0    0   0.0   0.0  0.0  0.0    0.0      0.0  1.0  \n",
       "4        0    0   0.0   0.0  0.0  0.0    0.0      0.0  1.0  \n",
       "...    ...  ...   ...   ...  ...  ...    ...      ...  ...  \n",
       "17557    0    0   0.0   0.0  1.0  0.0    0.0      0.0  0.0  \n",
       "17558    0    0   0.0   0.0  0.0  0.0    0.0      0.0  1.0  \n",
       "17559    0    0   0.0   0.0  0.0  0.0    0.0      0.0  1.0  \n",
       "17560    0    0   0.0   0.0  0.0  0.0    0.0      0.0  1.0  \n",
       "17561    0    0   0.0   0.0  0.0  0.0    0.0      0.0  1.0  \n",
       "\n",
       "[17562 rows x 87 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rating Ordinal Encoding\n",
    "\n",
    "Note that Unknown variables are also grouped along with the most severe rating, just in case. Don't want to accidentally recommend a Adult title if one of the Unknown rating is an adult title. It would be bad to recommend an adult title to the wrong audience"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['R - 17+ (violence & profanity)', 'PG-13 - Teens 13 or older',\n",
       "       'PG - Children', 'R+ - Mild Nudity', 'G - All Ages', 'Rx - Hentai',\n",
       "       'Unknown'], dtype=object)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['rating'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Order G to Adult from 1-6\n",
    "anime['rating'] = anime['rating'].replace({'G - All Ages': 1, 'PG - Children': 2, \n",
    "                                           'PG-13 - Teens 13 or older': 3, 'R+ - Mild Nudity': 4,\n",
    "                                           'R - 17+ (violence & profanity)': 5, 'Rx - Hentai': 6,\n",
    "                                           'Unknown': 6})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        5\n",
       "1        5\n",
       "2        3\n",
       "3        3\n",
       "4        2\n",
       "        ..\n",
       "17557    6\n",
       "17558    6\n",
       "17559    5\n",
       "17560    3\n",
       "17561    6\n",
       "Name: rating, Length: 17562, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime['rating']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merging dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First subset the anime table\n",
    "anime_cols = ['mal_id', 'score', 'episodes', 'rating', 'ranked', 'popularity',\n",
    "       'members', 'score_10', 'score_9', 'score_8', 'score_7', 'score_6',\n",
    "       'score_5', 'score_4', 'score_3', 'score_2', 'score_1',\n",
    "       'days_since_aired', 'Action', 'Adventure', 'Cars', 'Comedy', 'Dementia',\n",
    "       'Demons', 'Drama', 'Ecchi', 'Fantasy', 'Game', 'Harem', 'Hentai',\n",
    "       'Historical', 'Horror', 'Josei', 'Kids', 'Magic', 'Martial Arts',\n",
    "       'Mecha', 'Military', 'Music', 'Mystery', 'Parody', 'Police',\n",
    "       'Psychological', 'Romance', 'Samurai', 'School', 'Sci-Fi', 'Seinen',\n",
    "       'Shoujo', 'Shoujo Ai', 'Shounen', 'Shounen Ai', 'Slice of Life',\n",
    "       'Space', 'Sports', 'Super Power', 'Supernatural', 'Thriller', 'Unknown',\n",
    "       'Vampire', 'Yaoi', 'Yuri', 'Movie', 'Music', 'ONA', 'OVA', 'Other',\n",
    "       'Special', 'TV']\n",
    "\n",
    "anime_subset = anime[anime_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Global replace 'Unknown'\n",
    "anime_subset = anime_subset.replace({'Unknown': np.nan})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mal_id</th>\n",
       "      <th>score</th>\n",
       "      <th>episodes</th>\n",
       "      <th>rating</th>\n",
       "      <th>ranked</th>\n",
       "      <th>popularity</th>\n",
       "      <th>members</th>\n",
       "      <th>score_10</th>\n",
       "      <th>score_9</th>\n",
       "      <th>score_8</th>\n",
       "      <th>...</th>\n",
       "      <th>Yaoi</th>\n",
       "      <th>Yuri</th>\n",
       "      <th>Movie</th>\n",
       "      <th>Music</th>\n",
       "      <th>Music</th>\n",
       "      <th>ONA</th>\n",
       "      <th>OVA</th>\n",
       "      <th>Other</th>\n",
       "      <th>Special</th>\n",
       "      <th>TV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>8.780000</td>\n",
       "      <td>26</td>\n",
       "      <td>5</td>\n",
       "      <td>28.0</td>\n",
       "      <td>39</td>\n",
       "      <td>1251960</td>\n",
       "      <td>229170.0</td>\n",
       "      <td>182126.0</td>\n",
       "      <td>131625.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>8.390000</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>159.0</td>\n",
       "      <td>518</td>\n",
       "      <td>273145</td>\n",
       "      <td>30043.0</td>\n",
       "      <td>49201.0</td>\n",
       "      <td>49505.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>8.240000</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>266.0</td>\n",
       "      <td>201</td>\n",
       "      <td>558913</td>\n",
       "      <td>50229.0</td>\n",
       "      <td>75651.0</td>\n",
       "      <td>86142.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>7.270000</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>2481.0</td>\n",
       "      <td>1467</td>\n",
       "      <td>94683</td>\n",
       "      <td>2182.0</td>\n",
       "      <td>4806.0</td>\n",
       "      <td>10128.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>6.980000</td>\n",
       "      <td>52</td>\n",
       "      <td>2</td>\n",
       "      <td>3710.0</td>\n",
       "      <td>4369</td>\n",
       "      <td>13224</td>\n",
       "      <td>312.0</td>\n",
       "      <td>529.0</td>\n",
       "      <td>1242.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17557</th>\n",
       "      <td>48481</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13116</td>\n",
       "      <td>354</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17558</th>\n",
       "      <td>48483</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17562</td>\n",
       "      <td>7010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17559</th>\n",
       "      <td>48488</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17558</td>\n",
       "      <td>11309</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17560</th>\n",
       "      <td>48491</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17565</td>\n",
       "      <td>1386</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17561</th>\n",
       "      <td>48492</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17563</td>\n",
       "      <td>310</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17562 rows × 71 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       mal_id     score episodes  rating  ranked  popularity  members  \\\n",
       "0           1  8.780000       26       5    28.0          39  1251960   \n",
       "1           5  8.390000        1       5   159.0         518   273145   \n",
       "2           6  8.240000       26       3   266.0         201   558913   \n",
       "3           7  7.270000       26       3  2481.0        1467    94683   \n",
       "4           8  6.980000       52       2  3710.0        4369    13224   \n",
       "...       ...       ...      ...     ...     ...         ...      ...   \n",
       "17557   48481  6.509999      NaN       6     NaN       13116      354   \n",
       "17558   48483  6.509999      NaN       6     NaN       17562     7010   \n",
       "17559   48488  6.509999      NaN       5     NaN       17558    11309   \n",
       "17560   48491  6.509999      NaN       3     NaN       17565     1386   \n",
       "17561   48492  6.509999      NaN       6     NaN       17563      310   \n",
       "\n",
       "       score_10   score_9   score_8  ... Yaoi Yuri Movie Music Music  ONA  \\\n",
       "0      229170.0  182126.0  131625.0  ...    0    0   0.0     0   0.0  0.0   \n",
       "1       30043.0   49201.0   49505.0  ...    0    0   1.0     0   0.0  0.0   \n",
       "2       50229.0   75651.0   86142.0  ...    0    0   0.0     0   0.0  0.0   \n",
       "3        2182.0    4806.0   10128.0  ...    0    0   0.0     0   0.0  0.0   \n",
       "4         312.0     529.0    1242.0  ...    0    0   0.0     0   0.0  0.0   \n",
       "...         ...       ...       ...  ...  ...  ...   ...   ...   ...  ...   \n",
       "17557       NaN       NaN       NaN  ...    0    0   0.0     0   0.0  1.0   \n",
       "17558       NaN       NaN       NaN  ...    0    0   0.0     0   0.0  0.0   \n",
       "17559       1.0       NaN       NaN  ...    0    0   0.0     0   0.0  0.0   \n",
       "17560       NaN       NaN       NaN  ...    0    0   0.0     0   0.0  0.0   \n",
       "17561       NaN       NaN       NaN  ...    0    0   0.0     0   0.0  0.0   \n",
       "\n",
       "       OVA  Other  Special   TV  \n",
       "0      0.0    0.0      0.0  1.0  \n",
       "1      0.0    0.0      0.0  0.0  \n",
       "2      0.0    0.0      0.0  1.0  \n",
       "3      0.0    0.0      0.0  1.0  \n",
       "4      0.0    0.0      0.0  1.0  \n",
       "...    ...    ...      ...  ...  \n",
       "17557  0.0    0.0      0.0  0.0  \n",
       "17558  0.0    0.0      0.0  1.0  \n",
       "17559  0.0    0.0      0.0  1.0  \n",
       "17560  0.0    0.0      0.0  1.0  \n",
       "17561  0.0    0.0      0.0  1.0  \n",
       "\n",
       "[17562 rows x 71 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime_subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "anime_subset['episodes'] = anime_subset['episodes'].fillna(anime_subset['episodes'].median())\n",
    "anime_subset['ranked'] = anime_subset['ranked'].fillna(anime_subset['ranked'].median())\n",
    "\n",
    "for s in range(1, 11):\n",
    "    anime_subset['score_' + str(s)] = anime_subset['score_' + str(s)].fillna(anime_subset['score_' + str(s)].median())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "mal_id      0\n",
       "score       0\n",
       "episodes    0\n",
       "rating      0\n",
       "ranked      0\n",
       "           ..\n",
       "ONA         0\n",
       "OVA         0\n",
       "Other       0\n",
       "Special     0\n",
       "TV          0\n",
       "Length: 71, dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime_subset.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename rating in anime_list to prevent confusion\n",
    "anime_list.rename(columns={'rating': 'user_rating'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "anime_list = pd.merge(anime_list, watching_status, left_on='watching_status', right_on='status')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>anime_id</th>\n",
       "      <th>user_rating</th>\n",
       "      <th>watching_status</th>\n",
       "      <th>watched_episodes</th>\n",
       "      <th>status</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>336145</td>\n",
       "      <td>2251</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>343247</td>\n",
       "      <td>2787</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>196188</td>\n",
       "      <td>22729</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>161132</td>\n",
       "      <td>1119</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>175224</td>\n",
       "      <td>20909</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999991</th>\n",
       "      <td>315329</td>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999992</th>\n",
       "      <td>20341</td>\n",
       "      <td>1887</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999993</th>\n",
       "      <td>5720</td>\n",
       "      <td>18679</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999994</th>\n",
       "      <td>291821</td>\n",
       "      <td>4898</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999995</th>\n",
       "      <td>173775</td>\n",
       "      <td>11785</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>999996 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  anime_id  user_rating  watching_status  watched_episodes  \\\n",
       "0        336145      2251           10                2                13   \n",
       "1        343247      2787            9                2                24   \n",
       "2        196188     22729            8                2                12   \n",
       "3        161132      1119            7                2                 1   \n",
       "4        175224     20909            6                2                 7   \n",
       "...         ...       ...          ...              ...               ...   \n",
       "999991   315329        16            6                4                 9   \n",
       "999992    20341      1887            0                4                 3   \n",
       "999993     5720     18679            4                4                 1   \n",
       "999994   291821      4898            0                4                 0   \n",
       "999995   173775     11785            0                4                 2   \n",
       "\n",
       "        status description  \n",
       "0            2   Completed  \n",
       "1            2   Completed  \n",
       "2            2   Completed  \n",
       "3            2   Completed  \n",
       "4            2   Completed  \n",
       "...        ...         ...  \n",
       "999991       4     Dropped  \n",
       "999992       4     Dropped  \n",
       "999993       4     Dropped  \n",
       "999994       4     Dropped  \n",
       "999995       4     Dropped  \n",
       "\n",
       "[999996 rows x 7 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>anime_id</th>\n",
       "      <th>user_rating</th>\n",
       "      <th>watching_status</th>\n",
       "      <th>watched_episodes</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>336145</td>\n",
       "      <td>2251</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>343247</td>\n",
       "      <td>2787</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>196188</td>\n",
       "      <td>22729</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>161132</td>\n",
       "      <td>1119</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>175224</td>\n",
       "      <td>20909</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>Completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999991</th>\n",
       "      <td>315329</td>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999992</th>\n",
       "      <td>20341</td>\n",
       "      <td>1887</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999993</th>\n",
       "      <td>5720</td>\n",
       "      <td>18679</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999994</th>\n",
       "      <td>291821</td>\n",
       "      <td>4898</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999995</th>\n",
       "      <td>173775</td>\n",
       "      <td>11785</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>Dropped</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>999996 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  anime_id  user_rating  watching_status  watched_episodes  \\\n",
       "0        336145      2251           10                2                13   \n",
       "1        343247      2787            9                2                24   \n",
       "2        196188     22729            8                2                12   \n",
       "3        161132      1119            7                2                 1   \n",
       "4        175224     20909            6                2                 7   \n",
       "...         ...       ...          ...              ...               ...   \n",
       "999991   315329        16            6                4                 9   \n",
       "999992    20341      1887            0                4                 3   \n",
       "999993     5720     18679            4                4                 1   \n",
       "999994   291821      4898            0                4                 0   \n",
       "999995   173775     11785            0                4                 2   \n",
       "\n",
       "       description  \n",
       "0        Completed  \n",
       "1        Completed  \n",
       "2        Completed  \n",
       "3        Completed  \n",
       "4        Completed  \n",
       "...            ...  \n",
       "999991     Dropped  \n",
       "999992     Dropped  \n",
       "999993     Dropped  \n",
       "999994     Dropped  \n",
       "999995     Dropped  \n",
       "\n",
       "[999996 rows x 6 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anime_list = anime_list.drop(columns=['status'])\n",
    "anime_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the anime_subset dataframe\n",
    "\n",
    "This will be used for KNN for title recommendations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "anime_subset.to_csv('Data/anime_subset.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge anime_subset to anime_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_df = pd.merge(anime_list, anime_subset, left_on='anime_id', right_on='mal_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_df.drop(columns=['description'], inplace=True)\n",
    "# Rename Score to aggregated score\n",
    "processed_df.rename(columns={'score': 'aggregated_score'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>anime_id</th>\n",
       "      <th>user_rating</th>\n",
       "      <th>watching_status</th>\n",
       "      <th>watched_episodes</th>\n",
       "      <th>mal_id</th>\n",
       "      <th>aggregated_score</th>\n",
       "      <th>episodes</th>\n",
       "      <th>rating</th>\n",
       "      <th>ranked</th>\n",
       "      <th>...</th>\n",
       "      <th>Yaoi</th>\n",
       "      <th>Yuri</th>\n",
       "      <th>Movie</th>\n",
       "      <th>Music</th>\n",
       "      <th>Music</th>\n",
       "      <th>ONA</th>\n",
       "      <th>OVA</th>\n",
       "      <th>Other</th>\n",
       "      <th>Special</th>\n",
       "      <th>TV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>336145</td>\n",
       "      <td>2251</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>2251</td>\n",
       "      <td>8.420000</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>143.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>228773</td>\n",
       "      <td>2251</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>2251</td>\n",
       "      <td>8.420000</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>143.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>61090</td>\n",
       "      <td>2251</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>2251</td>\n",
       "      <td>8.420000</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>143.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>26848</td>\n",
       "      <td>2251</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>2251</td>\n",
       "      <td>8.420000</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>143.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>195185</td>\n",
       "      <td>2251</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>2251</td>\n",
       "      <td>8.420000</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>143.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999991</th>\n",
       "      <td>300428</td>\n",
       "      <td>13503</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>13503</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>13708.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999992</th>\n",
       "      <td>30459</td>\n",
       "      <td>34369</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>34369</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>14403.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999993</th>\n",
       "      <td>275530</td>\n",
       "      <td>7779</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>7779</td>\n",
       "      <td>6.210000</td>\n",
       "      <td>52</td>\n",
       "      <td>2</td>\n",
       "      <td>7295.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999994</th>\n",
       "      <td>272555</td>\n",
       "      <td>40406</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>40406</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>15754.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999995</th>\n",
       "      <td>300428</td>\n",
       "      <td>35037</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>35037</td>\n",
       "      <td>6.509999</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11589.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>999996 rows × 76 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  anime_id  user_rating  watching_status  watched_episodes  \\\n",
       "0        336145      2251           10                2                13   \n",
       "1        228773      2251            9                2                13   \n",
       "2         61090      2251           10                2                13   \n",
       "3         26848      2251            9                2                13   \n",
       "4        195185      2251            8                2                13   \n",
       "...         ...       ...          ...              ...               ...   \n",
       "999991   300428     13503            0                4                 0   \n",
       "999992    30459     34369            0                4                 0   \n",
       "999993   275530      7779            0                4                 0   \n",
       "999994   272555     40406            0                4                 0   \n",
       "999995   300428     35037            0                4                 0   \n",
       "\n",
       "        mal_id  aggregated_score episodes  rating   ranked  ...  Yaoi  Yuri  \\\n",
       "0         2251          8.420000       13       5    143.0  ...     0     0   \n",
       "1         2251          8.420000       13       5    143.0  ...     0     0   \n",
       "2         2251          8.420000       13       5    143.0  ...     0     0   \n",
       "3         2251          8.420000       13       5    143.0  ...     0     0   \n",
       "4         2251          8.420000       13       5    143.0  ...     0     0   \n",
       "...        ...               ...      ...     ...      ...  ...   ...   ...   \n",
       "999991   13503          6.509999       11       3  13708.0  ...     0     0   \n",
       "999992   34369          6.509999        5       3  14403.0  ...     0     0   \n",
       "999993    7779          6.210000       52       2   7295.0  ...     0     0   \n",
       "999994   40406          6.509999       16       1  15754.0  ...     0     0   \n",
       "999995   35037          6.509999        1       1  11589.0  ...     0     0   \n",
       "\n",
       "       Movie Music Music  ONA  OVA Other Special   TV  \n",
       "0        0.0     0   0.0  0.0  0.0   0.0     0.0  1.0  \n",
       "1        0.0     0   0.0  0.0  0.0   0.0     0.0  1.0  \n",
       "2        0.0     0   0.0  0.0  0.0   0.0     0.0  1.0  \n",
       "3        0.0     0   0.0  0.0  0.0   0.0     0.0  1.0  \n",
       "4        0.0     0   0.0  0.0  0.0   0.0     0.0  1.0  \n",
       "...      ...   ...   ...  ...  ...   ...     ...  ...  \n",
       "999991   0.0     0   0.0  0.0  0.0   0.0     0.0  1.0  \n",
       "999992   0.0     0   0.0  0.0  0.0   0.0     1.0  0.0  \n",
       "999993   0.0     0   0.0  0.0  0.0   0.0     0.0  1.0  \n",
       "999994   0.0     0   0.0  0.0  0.0   0.0     0.0  1.0  \n",
       "999995   0.0     0   0.0  0.0  1.0   0.0     0.0  0.0  \n",
       "\n",
       "[999996 rows x 76 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save Merged and Cleaned Dataset to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save as csv file to S3\n",
    "processed_df.to_csv('Data/training.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def upload_file(file_name, bucket, object_name=None):\n",
    "    \"\"\"Upload a file to an S3 bucket\n",
    "\n",
    "    :param file_name: File to upload\n",
    "    :param bucket: Bucket to upload to\n",
    "    :param object_name: S3 object name. If not specified then file_name is used\n",
    "    :return: True if file was uploaded, else False\n",
    "    \"\"\"\n",
    "\n",
    "    # If S3 object_name was not specified, use file_name\n",
    "    if object_name is None:\n",
    "        object_name = os.path.basename(file_name)\n",
    "\n",
    "    # Upload the file\n",
    "    s3_client = boto3.client('s3')\n",
    "    try:\n",
    "        response = s3_client.upload_file(file_name, bucket, object_name)\n",
    "    except ClientError as e:\n",
    "        logging.error(e)\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "upload_file('Data/anime_subset.csv', 'ads508team3', 'modeling/anime_subset.csv')\n",
    "upload_file('Data/training.csv', 'ads508team3', 'modeling/training.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "instance_type": "ml.m5.large",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}