fasttext_amazon_reviews/fasttext_amazon_reviews.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finding label issues in Amazon Reviews (Text Data) with FastText"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/cleanlab/examples/blob/master/fasttext_amazon_reviews/fasttext_amazon_reviews.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This example demonstrates how to find label issues using FastText modules compatible with `cleanlab`.\n",
    "\n",
    "Code is adapted from v1 examples (see `contrib/v1` folder)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please install the dependencies specified in this [requirements.txt](https://github.com/cleanlab/examples/blob/master/fasttext_amazon_reviews/requirements.txt) file before running the notebook."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the follow [bash script](https://github.com/cleanlab/examples/blob/master/fasttext_amazon_reviews/download_data.sh) to download all the data.\n",
    "\n",
    "```console\n",
    "$ bash ./download_data.sh\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from cleanlab.models.fasttext import FastTextClassifier, data_loader\n",
    "import cleanlab\n",
    "import numpy as np\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "import os\n",
    "from datetime import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# can skip this cell unless you changed the default data path\n",
    "data_dir = \"\"\n",
    "data_fn = \"\"\n",
    "write_fn = \"amazon5core.txt\"  # unprocessed data\n",
    "write_dir = \"data/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filter the original Amazon Reviews dataset:\n",
    "\n",
    "The dataset used in this tutorial is a subset of the original Amazon Reviews dataset filtered by the following items:\n",
    "\n",
    "1. coreset (reviewers who reviewed at least five things and products with at least five reviews)\n",
    "2. helpful (reviews with more helpful upvotes than unhelpful upvotes - requires at least one upvote)\n",
    "3. sentiment non-ambiguity (has to be rated 1, 3, or 5 -- it is hard to verify what a rating of 2 or 4 really means, ratings are better classified as positive, middle or negative, so all 2s and 4s are dropped)\n",
    "4. non-empty\n",
    "\n",
    "This results in ~ 10 million reviews."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetch and preprocess data\n",
    "\n",
    "need_text_data = False\n",
    "need_to_prepare_data_for_fasttext = False\n",
    "if need_to_prepare_data_for_fasttext:\n",
    "    # Convert amazon dataset to fasttext format\n",
    "    # Only include reviews with more helpful votes than unhelpful votes\n",
    "    # This takes about 6 minutes.\n",
    "    labels = []\n",
    "    with open(data_dir + data_fn, \"r\") as rf:\n",
    "        with open(write_dir + write_fn, \"w\") as wf:\n",
    "            #             for i in range(1000000):\n",
    "            #                 d = json.loads(rf.readline())\n",
    "            for line in rf:\n",
    "                d = json.loads(line)\n",
    "                h = d[\"helpful\"]\n",
    "                if h[0] > h[1] // 2:\n",
    "                    label = int(d[\"overall\"])\n",
    "                    if label in [1, 3, 5]:\n",
    "                        text = d[\"reviewText\"]\n",
    "                        if len(text) > 0:\n",
    "                            wf.write(\n",
    "                                \"__label__{} {}\\n\".format(\n",
    "                                    label,\n",
    "                                    text.strip().replace(\"\\n\", \" __newline__ \"),\n",
    "                                )\n",
    "                            )\n",
    "                            labels.append(label)\n",
    "    label_map = {1: 0, 3: 1, 5: 2}\n",
    "    labels = [label_map[l] for l in labels]\n",
    "else:\n",
    "    labels = np.empty(9996437, dtype=int)\n",
    "    if need_text_data:\n",
    "        text = []\n",
    "    loc = write_dir + \"amazon5core.preprocessed.txt\"\n",
    "    bs = 1000000\n",
    "    label_map = {\"__label__1\": 0, \"__label__3\": 1, \"__label__5\": 2}\n",
    "    for i, (l, t) in enumerate(data_loader(loc, batch_size=bs)):\n",
    "        labels[bs * i : bs * (i + 1)] = [label_map[lab] for lab in l]\n",
    "        if need_text_data:\n",
    "            text.append(t)\n",
    "    if need_text_data:\n",
    "        text = [t for lst in text for t in lst]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we find the best parameters for the FastText model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_list = ParameterGrid(\n",
    "    {\n",
    "        \"cv_n_folds\": [3],\n",
    "        \"lr\": [0.01, 0.05, 0.1, 0.5, 1.0],\n",
    "        \"ngram\": [3],\n",
    "        \"epochs\": [1, 5, 10],\n",
    "        \"dim\": [100],\n",
    "    }\n",
    ")\n",
    "seed = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Fasttext model selection.\n",
    "\n",
    "start_time = dt.now()\n",
    "scores = []\n",
    "for i, params in enumerate(param_list):\n",
    "    print(params)\n",
    "    if i > 0:\n",
    "        elapsed = dt.now() - start_time\n",
    "        total_time = elapsed * len(param_list) / float(i)\n",
    "        remaining = total_time - elapsed\n",
    "        print(\"Elapsed:\", str(elapsed)[:-7], \"| Remaining:\", str(remaining)[:-7])\n",
    "    ftc = FastTextClassifier(\n",
    "        train_data_fn=write_dir + \"amazon5core.preprocessed.txt\",\n",
    "        batch_size=100000,\n",
    "        labels=[1, 3, 5],\n",
    "        kwargs_train_supervised={\n",
    "            \"epoch\": params[\"epochs\"],\n",
    "            \"thread\": 12,\n",
    "            \"lr\": params[\"lr\"],\n",
    "            \"wordNgrams\": params[\"ngram\"],\n",
    "            \"bucket\": 200000,\n",
    "            \"dim\": params[\"dim\"],\n",
    "            \"loss\": \"softmax\",  #'softmax', # 'hs'\n",
    "        },\n",
    "    )\n",
    "    pyx = cleanlab.count.estimate_cv_predicted_probabilities(\n",
    "        X=np.arange(len(labels)),\n",
    "        labels=labels,\n",
    "        clf=ftc,\n",
    "        cv_n_folds=params[\"cv_n_folds\"],\n",
    "        seed=seed,\n",
    "    )\n",
    "    # Write out\n",
    "    wfn = (\n",
    "        write_dir\n",
    "        + \"amazon_pyx_cv__folds_{}__epochs_{}__lr_{}__ngram_{}__dim_{}.npy\".format(\n",
    "            params[\"cv_n_folds\"],\n",
    "            params[\"epochs\"],\n",
    "            params[\"lr\"],\n",
    "            params[\"ngram\"],\n",
    "            params[\"dim\"],\n",
    "        )\n",
    "    )\n",
    "    with open(wfn, \"wb\") as wf:\n",
    "        np.save(wf, pyx)\n",
    "\n",
    "    # Check that probabilities are good.\n",
    "    print(\"pyx finished. Writing:\", wfn)\n",
    "    scores.append(accuracy_score(labels, np.argmax(pyx, axis=1)))\n",
    "    print(\"Acc:\", np.round(scores[-1], 4))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params = param_list[np.argmax(scores)]\n",
    "print(\"best params\", best_params)\n",
    "wfn = (\n",
    "    write_dir\n",
    "    + \"amazon_pyx_cv__folds_{}__epochs_{}__lr_{}__ngram_{}__dim_{}.npy\".format(\n",
    "        params[\"cv_n_folds\"],\n",
    "        params[\"epochs\"],\n",
    "        params[\"lr\"],\n",
    "        params[\"ngram\"],\n",
    "        params[\"dim\"],\n",
    "    )\n",
    ")\n",
    "print(\"located in:\", wfn)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lastly, we train the best model and view its performace."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_from_scratch = True\n",
    "\n",
    "# Train the best model from scratch\n",
    "# No need to do this if you've already run the\n",
    "# hyper-parameter optimization above.\n",
    "if train_from_scratch:\n",
    "    cv_n_folds = 10  # Increasing more improves pyx, at great cost.\n",
    "    seed = 0\n",
    "    lr = 0.01\n",
    "    ngram = 3\n",
    "    epochs = 10  # Increasing more doesn't do much.\n",
    "    dim = 100\n",
    "\n",
    "    ftc = FastTextClassifier(\n",
    "        train_data_fn=write_dir + \"amazon5core.preprocessed.txt\",\n",
    "        batch_size=100000,\n",
    "        labels=[1, 3, 5],\n",
    "        kwargs_train_supervised={\n",
    "            \"epoch\": epochs,\n",
    "            \"thread\": 12,\n",
    "            \"lr\": lr,\n",
    "            \"wordNgrams\": ngram,\n",
    "            \"bucket\": 200000,\n",
    "            \"dim\": dim,\n",
    "            \"loss\": \"softmax\",  #'softmax', # 'hs'\n",
    "        },\n",
    "    )\n",
    "\n",
    "    pyx = cleanlab.count.estimate_cv_predicted_probabilities(\n",
    "        X=np.arange(len(labels)),\n",
    "        labels=labels,\n",
    "        clf=ftc,\n",
    "        cv_n_folds=cv_n_folds,\n",
    "        seed=seed,\n",
    "    )\n",
    "\n",
    "    # Write out pyx\n",
    "    wfn = (\n",
    "        write_dir\n",
    "        + \"amazon_pyx_cv__folds_{}__epochs_{}__lr_{}__ngram_{}__dim_{}.npy\".format(\n",
    "            cv_n_folds, epochs, lr, ngram, dim\n",
    "        )\n",
    "    )\n",
    "    with open(wfn, \"wb\") as wf:\n",
    "        np.save(wf, pyx)\n",
    "\n",
    "    # Check that probabilities are good.\n",
    "    print(wfn)\n",
    "    accuracy_score(labels, np.argmax(pyx, axis=1))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "00885e89789f58e60dbba52a405dc834aaf92411914fde0d391f9b48289a0610"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}