homework2_student_ver.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    },
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Problem 1: Modeling with ClinicalBERT Embeddings"
      ],
      "metadata": {
        "id": "U93wKjNTK5ET"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note: to avoid consuming the GPU resources allocated to you by colab on the parts you don't need a GPU for, make sure you use a CPU runtime (Runtime > Change Runtime Type > Hardware accelerator: None) until the notebook indicates otherwise."
      ],
      "metadata": {
        "id": "1gv0j1MLfBrN"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install Libraries"
      ],
      "metadata": {
        "id": "BUmfHV_VK9g9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "!pip install scikit-learn\n",
        "!pip install umap-learn"
      ],
      "metadata": {
        "id": "Tv3lgtR4nEsM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setting up Google Drive\n",
        "Copy the data at the [following link](https://drive.google.com/drive/folders/1G5NuAnUSaKzcry-tzgPZKxafG_vcOzX9?usp=sharing) to a folder in your own drive and set the path to that folder below"
      ],
      "metadata": {
        "id": "hfVoxO-7BBuA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Path to saved data\n",
        "#------YOUR CODE HERE--------\n",
        "data_path = \"/content/drive/MyDrive/path_to_your_folder\"\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "hNe_F-2HBAdB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9j60f3KrfsG"
      },
      "source": [
        "from google.colab import auth, drive\n",
        "drive.mount('/content/drive')\n",
        "auth.authenticate_user()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7qfVkr1MzXgu"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os \n",
        "import random\n",
        "import sklearn\n",
        "import importlib\n",
        "import pickle\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import time\n",
        "\n",
        "from pathlib import Path\n",
        "from torch.utils import data\n",
        "from tqdm.auto import tqdm\n",
        "from transformers import AutoTokenizer, AutoModel, BertConfig, BertTokenizer, BertForMaskedLM, InputExample\n",
        "\n",
        "pd.set_option('display.max_columns', 50)\n",
        "pd.options.mode.chained_assignment = None\n",
        "\n",
        "# Add random seed\n",
        "random.seed(456)\n",
        "np.random.seed(456)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(a) You can read more about ClinicalBERT [here](https://huggingface.co/emilyalsentzer/Bio_ClinicalBERT)"
      ],
      "metadata": {
        "id": "ZLNCV2jvqcpP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Initialize the tokenizer\n",
        "tokenizer = \n",
        "\n",
        "# Initialize the model\n",
        "model = \n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "sqZxtqxpMmg_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "metadata": {
        "id": "J1v0gkPstnaG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(b)"
      ],
      "metadata": {
        "id": "mEjK57qaLCV7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def fill_blank(text: str, model: BertForMaskedLM, tokenizer: BertTokenizer) -> str:\n",
        "    '''\n",
        "    Given a sentence with a single blank (denoted by an underscore), queries the BERT model to \n",
        "        fill in the missing token.\n",
        "        \n",
        "    Inputs:\n",
        "        - text: sentence containing a single underscore corresponding to the missing token\n",
        "        - model: pytorch ClinicalBERT model, of type BertForMaskedLM\n",
        "        - tokenizer: BertTokenizer object\n",
        "    \n",
        "    Output:\n",
        "        - string corresponding to the sentence where the underscore is replaced with the most likely token\n",
        "    '''\n",
        "    random.seed(456)\n",
        "    np.random.seed(456)\n",
        "    torch.manual_seed(456)\n",
        "    \n",
        "    #------YOUR CODE HERE--------\n",
        "    # Replace the underscore by [MASK] and store the result in masked_str\n",
        "    masked_str = \n",
        "\n",
        "    # Tokenize the masked string and store the tokens in inputs\n",
        "    inputs = \n",
        "    #------YOUR CODE ENDS--------\n",
        "\n",
        "    inputs = inputs.to(device)\n",
        "    with torch.no_grad():\n",
        "        #------YOUR CODE HERE--------\n",
        "        # Compute the logits (log probabilities) from the model\n",
        "        logits = \n",
        "        #------YOUR CODE ENDS--------\n",
        "\n",
        "    mask_token_index = (inputs.input_ids == tokenizer.mask_token_id)[0].nonzero(as_tuple=True)[0]\n",
        "\n",
        "    predicted_logits = logits[0, mask_token_index] \n",
        "    #------YOUR CODE HERE--------\n",
        "    # Select the most likely token in predicted_logits\n",
        "    predicted_token_id = \n",
        "\n",
        "    # Use the tokenizer to decode the token id into a string\n",
        "    pred = \n",
        "    #------YOUR CODE ENDS--------\n",
        "\n",
        "    return text.replace('_', pred)"
      ],
      "metadata": {
        "id": "k9RM0cJxTVTe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Test fill_blank\n",
        "nurse_sent = '30 yo white _ helping other nurses at the ICU'\n",
        "doc_sent = '30 yo white _ helping other doctors at the ICU'\n",
        "print(f\"Predicted sentence: {fill_blank(nurse_sent, model, tokenizer)}\")\n",
        "print(\"Expected sentence: 30 yo white female helping other nurses at the ICU\")\n",
        "print(f\"Predicted sentence: {fill_blank(doc_sent, model, tokenizer)}\")\n",
        "print(\"Expected sentence: 30 yo white male helping other doctors at the ICU\")"
      ],
      "metadata": {
        "id": "hwadavOxsn7v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(c)"
      ],
      "metadata": {
        "id": "lReDVacVwAPU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "sent1 = 'Sentence 1 goes here!'\n",
        "sent2 = 'Sentence 2 goes here!'\n",
        "#------YOUR CODE ENDS--------\n",
        "print(f\"Sentence 1 (completed): {fill_blank(sent1, model, tokenizer)}\")\n",
        "print(f\"Sentence 2 (completed): {fill_blank(sent2, model, tokenizer)}\")"
      ],
      "metadata": {
        "id": "yxnT3yzFwSFL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(d) Answer in your report"
      ],
      "metadata": {
        "id": "4XX3lVy3xDAm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "(e)"
      ],
      "metadata": {
        "id": "4bcW_vqkxfYR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Change the runtime to GPU for this part (Runtime > Change Runtime Type > Hardware Accelerator: GPU)"
      ],
      "metadata": {
        "id": "p87qzrBG1wOH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the data\n",
        "df = pd.read_hdf(os.path.join(data_path, \"text_and_hypertension_data.h5\"))"
      ],
      "metadata": {
        "id": "OPLJobKBp1n3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_sent_rep(model, tokenizer, txt):\n",
        "    \"\"\"\n",
        "    Compute the sentence representation and return it as a numpy array\n",
        "    If done correctly, the numpy array should be of size 768\n",
        "    \"\"\"\n",
        "    # Tokenize the input txt and store the result in inputs\n",
        "    # Remember to set truncation=True and max_length=512\n",
        "    #------YOUR CODE HERE--------\n",
        "    inputs = \n",
        "    #------YOUR CODE ENDS--------\n",
        "    inputs.to(device)\n",
        "    model.to(device)\n",
        "    with torch.no_grad():\n",
        "        #------YOUR CODE HERE--------\n",
        "        # Compute the model outputs and store the result in outputs\n",
        "        # Make sure output_hidden_states=True\n",
        "        outputs = \n",
        "        #------YOUR CODE ENDS--------\n",
        "    \n",
        "        embed = outputs.hidden_states[-1]\n",
        "\n",
        "        #------YOUR CODE HERE--------\n",
        "        # embed (of size [1, input_length, 768])\n",
        "        # contains the hidden states corresponding to each\n",
        "        # token at the final layer of the model\n",
        "        # Each hidden state is a vector of size 768\n",
        "        # Compute the mean of these vectors to get a representation\n",
        "        # of the input sentence, and store the mean again in embed\n",
        "        embed = \n",
        "        #------YOUR CODE ENDS--------\n",
        "\n",
        "        embed = embed.squeeze()\n",
        "\n",
        "    return embed.cpu().detach().numpy()"
      ],
      "metadata": {
        "id": "tvYSo6JQtZW3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following block of code should take around 15min to run on a GPU the first time it is run. Make sure to save its result in your drive to avoid needing to run it again"
      ],
      "metadata": {
        "id": "eteA2T5q5D89"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "recompute_embeds = False\n",
        "# Only regenerate embeds if necessary\n",
        "if not os.path.exists(os.path.join(data_path, \"embeds.npy\")) or recompute_embeds == True:\n",
        "    # Generate embeddings\n",
        "    num_pts = len(df)\n",
        "    embeds = [None]* num_pts\n",
        "    start = time.time()\n",
        "    for row_idx in range(num_pts):\n",
        "        note_data = df.iloc[row_idx][\"text\"]\n",
        "        embeds[row_idx] = get_sent_rep(model, tokenizer, note_data)\n",
        "    print(time.time() - start)\n",
        "    X = np.stack(embeds, axis=0)\n",
        "    with open(os.path.join(data_path, \"embeds.npy\"), \"wb\") as f:\n",
        "        np.save(f, X)"
      ],
      "metadata": {
        "id": "U-wPQFJxPKAs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(f)"
      ],
      "metadata": {
        "id": "UhqQP6_O6U8s"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "You can change the runtime back to CPU to avoid using your GPU allocation"
      ],
      "metadata": {
        "id": "EwoanXGrHXNz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Get train and test data\n",
        "with open(os.path.join(data_path, \"embeds.npy\"), \"rb\") as f:\n",
        "    # X contains one embedding per row corresponding to\n",
        "    # the discharge summary of the patient in that row\n",
        "    # in the dataset\n",
        "    X = np.load(f)\n",
        "\n",
        "# y contains whether the patient in a particular row had\n",
        "# hypertension during their ICU stay\n",
        "y = df['Hypertension'].tolist()\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=456)"
      ],
      "metadata": {
        "id": "7jvUc6PJ_dCY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "# Scale the train data\n",
        "scaler = StandardScaler()\n",
        "scaler.fit(X_train)\n",
        "X_train_scaled = scaler.transform(X_train)"
      ],
      "metadata": {
        "id": "U95_xwRXlMUo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Create and fit your logistic regression model on the training data\n",
        "# Make sure to use multi_class = \"multinomial\" and class_weight=\"balanced\"\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------\n"
      ],
      "metadata": {
        "id": "Kcf-lDpoYnTJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(g)"
      ],
      "metadata": {
        "id": "CmjRSWlq9oRw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score\n",
        "#------YOUR CODE HERE--------\n",
        "# Compute the performance metrics on the training set\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "FgEyht5id7sI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Scale the test data\n",
        "X_test_scaled = scaler.transform(X_test)"
      ],
      "metadata": {
        "id": "irLR523pDKFI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Compute the performance metrics on the test set\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE HERE--------"
      ],
      "metadata": {
        "id": "oXKVHVJjDPFc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(h)"
      ],
      "metadata": {
        "id": "OYzN-ef390B7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from umap import UMAP"
      ],
      "metadata": {
        "id": "fUBiL55gJjLT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Use UMAP to project the scaled training data onto two dimensions\n",
        "# Make sure to use random_state=456\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "o6XaGQBlJlsg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "#------YOUR CODE HERE--------\n",
        "# Plot the UMAP embeddings on a scatter plot\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "xM4BPqFGLqz1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Use UMAP to project the scaled test data onto two dimensions\n",
        "# Make sure to use random_state=456\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "PnG1IehU-uQ5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Plot the UMAP embeddings on a scatter plot\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "S34GqU9Y-uQ6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(i) Answer in your report"
      ],
      "metadata": {
        "id": "plsPZnjI_IOI"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "(j)"
      ],
      "metadata": {
        "id": "Jg0jl8JE_QKC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis"
      ],
      "metadata": {
        "id": "yCgQWLHrQd1K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Use LDA to project the scaled training data onto a single dimension\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "fzUPm8HDRPfI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Plot the LDA embeddings on two histograms on the same plot\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "4NFllfE7RxfL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Use LDA to project the scaled test data onto a single dimension\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "oG4OWQEf_hXT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Plot the LDA embeddings on two histograms on the same plot\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "LBOb16cu_hXT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(k) Answer in your report"
      ],
      "metadata": {
        "id": "2D6ON4n6_w9K"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "(l) Answer in your report"
      ],
      "metadata": {
        "id": "ZtjpRNZD_y8N"
      }
    }
  ]
}