Bertopic_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "from umap import UMAP\n",
        "from hdbscan import HDBSCAN\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from nltk.corpus import stopwords\n",
        "from bertopic import BERTopic\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import string"
      ],
      "metadata": {
        "id": "0gd-qV7WT2ge"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "Iub0lQS0UExB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the dataset\n",
        "dirty_df = pd.read_csv('/content/drive/MyDrive/BestJournals.csv', index_col=0)\n",
        "dirty_df = dirty_df[dirty_df[\"Abstract\"].notna()]"
      ],
      "metadata": {
        "id": "5ypjZo_pxfto"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Remove duplicate entries as well as Journals that have the title pasted in the abstract\n",
        "\n",
        "def cleaning_abs(dirty_df):\n",
        "   \n",
        "    tmp_list = list()\n",
        "\n",
        "    for i in range(len(dirty_df)-1):\n",
        "        if dirty_df.iloc[i][\"Title\"] != dirty_df.iloc[i+1][\"Title\"]:\n",
        "            if dirty_df.iloc[i][\"Abstract\"] != dirty_df.iloc[i+1][\"Abstract\"]:\n",
        "                if dirty_df.iloc[i][\"Name\"] != dirty_df.iloc[i+1][\"Name\"]:\n",
        "                    tmp_list.append(dirty_df.iloc[i])\n",
        "   \n",
        "    df = pd.DataFrame(tmp_list, columns = dirty_df.columns)\n",
        "\n",
        "    df.reset_index(drop = True, inplace = True)\n",
        "   \n",
        "    return df\n",
        "\n",
        "def remove_title_from_abs(df):\n",
        "    exclist = '#$%*+/<=>@[\\]^_`{|}~'\n",
        "    table = str.maketrans('', '', exclist)\n",
        "\n",
        "    for i in range(len(df)-1):\n",
        "\n",
        "      tmp_old = df.iloc[i][\"Abstract\"]\n",
        "\n",
        "      tmp_new = tmp_old.translate(table)\n",
        "      tmp_new = tmp_new.strip()\n",
        "\n",
        "      df.at[i, \"Abstract\"] = tmp_new\n",
        "\n",
        "\n",
        "    tmp_list = list()\n",
        "\n",
        "    for i in range(len(df)-1):\n",
        "      if df.iloc[i][\"Title\"][:30] != df.iloc[i][\"Abstract\"][:30]:\n",
        "        tmp_list.append(df.iloc[i])\n",
        "\n",
        "    df = pd.DataFrame(tmp_list, columns = dirty_df.columns)\n",
        "\n",
        "    df.reset_index(drop = True, inplace = True)\n",
        "\n",
        "    return df\n",
        "\n",
        "df = cleaning_abs(dirty_df)\n",
        "df = remove_title_from_abs(df)\n",
        "docs = df.Abstract.tolist()\n",
        "\n",
        "df.to_csv('processed_df.csv')"
      ],
      "metadata": {
        "id": "DIkXPuWBxoZG"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "journal_counts = df.groupby(['Year', 'Journal']).size().reset_index(name='count')\n",
        "journal_pivot = journal_counts.pivot(index='Year', columns='Journal', values='count')\n",
        "journal_pivot = journal_pivot.fillna(0).astype(int)\n",
        "journal_pivot['Total'] = journal_pivot.sum(axis=1)\n",
        "journal_pivot.loc['Total']= journal_pivot.sum()\n",
        "journal_pivot.to_excel('journal_pivot.xlsx')"
      ],
      "metadata": {
        "id": "H2-dqouSCtVI"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Defines a custom vectorizer class to remove n-gram stop words\n",
        "\n",
        "class CustomVectorizer(CountVectorizer): \n",
        "       \n",
        "    stop_grams = []    \n",
        "    \n",
        "    def __init__(self, stop_grams = [], **opts):\n",
        "        super().__init__(**opts)\n",
        "        self.stop_grams = stop_grams\n",
        "    \n",
        "    def remove_ngrams(self, doc):\n",
        "        for stop_gram in self.stop_grams:\n",
        "            doc = doc.replace(stop_gram, \"\")\n",
        "        return doc\n",
        "    \n",
        "    # overwrite the build_analyzer method, allowing one to\n",
        "    # create a custom analyzer for the vectorizer\n",
        "    def build_analyzer(self):\n",
        "        \n",
        "        # load stop words using CountVectorizer's built in method\n",
        "        stop_words = list(self.get_stop_words())\n",
        "        \n",
        "        preprocessor = self.build_preprocessor()\n",
        "        tokenizer = self.build_tokenizer()\n",
        "        remove_ngrams = self.remove_ngrams\n",
        "        \n",
        "        \n",
        "        # create the analyzer that will be returned by this method\n",
        "        def analyser(doc):\n",
        "                \n",
        "            # apply the preprocessing and tokenzation steps\n",
        "            doc_clean = preprocessor(doc.lower())\n",
        "            \n",
        "            # remove phrase stopwords\n",
        "            doc_clean = remove_ngrams(doc)\n",
        "            \n",
        "            # tokenize using default tokenizer\n",
        "            tokens = tokenizer(doc_clean)            \n",
        "            \n",
        "            # use CountVectorizer's _word_ngrams built in method\n",
        "            # to remove stop words and extract n-grams\n",
        "            return(self._word_ngrams(tokens, stop_words))\n",
        "        \n",
        "        return(analyser)"
      ],
      "metadata": {
        "id": "Sd2OFU6GyBGW"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Remove stop words\n",
        "\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "\n",
        "stop_words = stopwords.words('english')\n",
        "\n",
        "custom_sw = ['mathrsfs', 'amsmath', 'shrink', 'citation', \n",
        "             'volume', 'journal', 'Issue', 'Volume',\n",
        "             'EarlyView', 'The', 'This', 'We', 'In', 'Journal', 'Philosophical', 'Studies', 'JW', \n",
        "             'Page', 'Australasian', '90', '2012', 'March', 'article', 'abstract', '737', 'any055',\n",
        "             'analys', '10', 'Analysis', 'Wittgensteins', 'Tractatus', 'No', 'available', 'Book']\n",
        "\n",
        "stop_words.extend(custom_sw)\n",
        "\n",
        "stop_grams = ['available citation', 'citation abstract',\n",
        "              'citation abstract', 'abstract available',\n",
        "              'citation analysis', 'doi 10', '53 doi',\n",
        "              'usa philosophical', '1093 analys', 'australasian journal',\n",
        "              'doi 10', 'research 104', '2022 philosophy',\n",
        "              'australasian philosophy', 'philosophy 89',\n",
        "              'philosophy 90', 'Philosophical Studies', '1573 0883', 'Australasian Journal', \n",
        "              'Philosophy 89', 'Philosophy 90', 'original publication', 'review article', 'Analysis 78', 'analys any055',\n",
        "              'article abstract', 'citation Analysis', '1093 analys', '10 1093', 'Book review', \n",
        "              'Phenomenological Research', 'Philosophy Phenomenological', 'Research Philosophy', '2022 Philosophy',\n",
        "              'phenomenological research', 'Page 676', 'available citation', 'No available', \n",
        "              'citation No', 'citation Book', 'Analysis 78', '1093 analys', 'Type Article', 'philosophy and phenomenological research',\n",
        "               'Philosophy and Phenomenological Research', 'Philosophy 104', '2011 Philosophy', 'no available', 'available No', 'available Book',\n",
        "               'Book available']\n",
        "\n",
        "\n",
        "vectorizer_model = CustomVectorizer(ngram_range=(1, 2), stop_words=list(stop_words), stop_grams = stop_grams)"
      ],
      "metadata": {
        "id": "L__3lXDPyJvN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_model = SentenceTransformer('all-mpnet-base-v2')"
      ],
      "metadata": {
        "id": "q1jMOk5cySXu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embeddings = embedding_model.encode(docs, show_progress_bar=True)\n",
        "np.save('embeddings.npy', embeddings)"
      ],
      "metadata": {
        "id": "UXeyqSm-yZRy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "umap_model = UMAP(n_neighbors=8, n_components=4, min_dist=0.0, metric='cosine', random_state=42)"
      ],
      "metadata": {
        "id": "V8rwkUJLyUlz"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hdbscan_model = HDBSCAN(min_cluster_size=33, min_samples=31,\n",
        "                        gen_min_span_tree=True,\n",
        "                        metric='euclidean',\n",
        "                        prediction_data=True,\n",
        "                        cluster_selection_method='eom')"
      ],
      "metadata": {
        "id": "KyqMzHKhyWZF"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_bert = BERTopic(\n",
        "    umap_model=umap_model,\n",
        "    hdbscan_model=hdbscan_model,\n",
        "    embedding_model=embedding_model,\n",
        "    vectorizer_model=vectorizer_model,\n",
        "    top_n_words=15,\n",
        "    language='english',\n",
        "    calculate_probabilities=True,\n",
        "    verbose=True,\n",
        "    diversity=0.35\n",
        ")"
      ],
      "metadata": {
        "id": "u6sVe7uTycoE"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "topics, probs = model_bert.fit_transform(docs, embeddings)"
      ],
      "metadata": {
        "id": "6mMsCjIQyg6s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ecf6b56f-5873-4ad9-fc26-0f51649df09c"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2023-03-08 09:40:04,987 - BERTopic - Reduced dimensionality\n",
            "2023-03-08 09:40:11,381 - BERTopic - Clustered reduced embeddings\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_bert.save(\"topic_model\", save_embedding_model=False)"
      ],
      "metadata": {
        "id": "yNbRrsmDy5jw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "It is important to keep in mind that the same Python environment, as the one where the model was saved in, is needed when loading the model."
      ],
      "metadata": {
        "id": "IMwbrpFnzJEW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Loading a model on CPU only hardware\n",
        "# Additionally, define the CustomVectorizer again\n",
        "\n",
        "from bertopic.backend._utils import select_backend\n",
        "from sentence_transformers import SentenceTransformer\n",
        "\n",
        "embedding_model = SentenceTransformer('all-mpnet-base-v2')\n",
        "model = select_backend(embedding_model)\n",
        "model_bert = BERTopic.load(\"/content/drive/MyDrive/topic_model\", embedding_model = model)"
      ],
      "metadata": {
        "id": "FeEQWtCYzFgJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a clustering metric for model comparison\n",
        "# For in-depth information refer to: https://www.dbs.ifi.lmu.de/~zimek/publications/SDM2014/DBCV.pdf \n",
        "\n",
        "model_bert.hdbscan_model.relative_validity_"
      ],
      "metadata": {
        "id": "-d899diByiLa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all topics created by the model \n",
        "model_bert.get_topics()"
      ],
      "metadata": {
        "id": "p3UQ8E9uyuPy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get topic sizes\n",
        "model_bert.topic_sizes_"
      ],
      "metadata": {
        "id": "fFwmSyHeyw10"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get similar topics to given word\n",
        "similar_topics, similarity = model_bert.find_topics(\"statistics\", top_n = 3)\n",
        "\n",
        "most_similar = similar_topics[0]\n",
        "print(\"Most Similar Topic Info: \\n{}\".format(model_bert.get_topic(most_similar)))\n",
        "print(\"Similarity Score: {}\".format(similarity[0]))"
      ],
      "metadata": {
        "id": "NxNFZrEz0ZxW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Visualizations:"
      ],
      "metadata": {
        "id": "_O9fBv3PzjWF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Topic per class (journal)\n",
        "classes = list(df[\"Journal\"])\n",
        "topics_per_class = model_bert.topics_per_class(docs, classes=classes)"
      ],
      "metadata": {
        "id": "hJi-k3xgy3b6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6866aa53-2380-4763-afd8-b15d6ac0f7e5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "11it [01:15,  6.91s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Document cluster map\n",
        "model_bert.visualize_documents(docs, embeddings=embeddings)"
      ],
      "metadata": {
        "id": "NcOOY2ivzyqS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Similarity matrix\n",
        "model_bert.visualize_heatmap()"
      ],
      "metadata": {
        "id": "UhJ7j1hKz4jB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Topics over time\n",
        "timestamps = df[\"Year\"].to_list()\n",
        "topics_over_time = model_bert.topics_over_time(docs, timestamps, global_tuning=True, evolution_tuning=True)\n",
        "model_bert.visualize_topics_over_time(topics_over_time, top_n_topics=10, )"
      ],
      "metadata": {
        "id": "LQpjbHy6z8l3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Intertopic Distance Map\n",
        "model_bert.visualize_topics()"
      ],
      "metadata": {
        "id": "n5PaHp_X0Vle"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Parameter fine-tuning for UMAP and HDBSCAN.\n",
        "The negative DBCV for HDBSCAN was used as cost function to minimize."
      ],
      "metadata": {
        "id": "aW7ogSjU0hu_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, partial, space_eval"
      ],
      "metadata": {
        "id": "ILYessMw07SC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import umap\n",
        "import hdbscan\n",
        "\n",
        "def generate_clusters(message_embeddings,\n",
        "                      n_neighbors,\n",
        "                      n_components, \n",
        "                      min_cluster_size,\n",
        "                      min_samples,\n",
        "                      random_state = None):\n",
        "    \"\"\"\n",
        "    Generate HDBSCAN cluster object after reducing embedding dimensionality with UMAP\n",
        "    \"\"\"\n",
        "    \n",
        "    umap_embeddings = (umap.UMAP(n_neighbors=n_neighbors, \n",
        "                                n_components=n_components, \n",
        "                                min_dist = 0.0,\n",
        "                                metric='cosine', \n",
        "                                random_state=random_state)\n",
        "                            .fit_transform(message_embeddings))\n",
        "\n",
        "    clusters = hdbscan.HDBSCAN(min_cluster_size = min_cluster_size,\n",
        "                               min_samples = min_samples,\n",
        "                               metric='euclidean', \n",
        "                               gen_min_span_tree=True,\n",
        "                               cluster_selection_method='eom').fit(umap_embeddings)\n",
        "\n",
        "    return clusters"
      ],
      "metadata": {
        "id": "aT007Jkm0m9D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "In this approach points are not assigned cluster labels, but are instead assigned a vector of probabilities. \n",
        "The length of the vector is equal to the number of clusters found. \n",
        "The probability value at the ith entry of the vector is the probability that that point is a member of the ith cluster\n",
        "'''\n",
        "def score_clusters(clusters, prob_threshold = 0.05):\n",
        "    \"\"\"\n",
        "    Returns the label count and cost of a given cluster supplied from running hdbscan\n",
        "    \"\"\"\n",
        "    \n",
        "    cluster_labels = clusters.labels_\n",
        "    label_count = len(np.unique(cluster_labels))\n",
        "    total_num = len(clusters.labels_)\n",
        "    cost = -clusters.relative_validity_\n",
        "    \n",
        "    return label_count, cost"
      ],
      "metadata": {
        "id": "m_nJtq5V0xhx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import trange\n",
        "import sklearn\n",
        "import random\n",
        "\n",
        "def random_search(embeddings, space, num_evals):\n",
        "    \"\"\"\n",
        "    Randomly search hyperparameter space and limited number of times \n",
        "    and return a summary of the results\n",
        "    \"\"\"\n",
        "    \n",
        "    results = []\n",
        "    \n",
        "    for i in trange(num_evals):\n",
        "        n_neighbors = random.choice(space['n_neighbors'])\n",
        "        n_components = random.choice(space['n_components'])\n",
        "        min_cluster_size = random.choice(space['min_cluster_size'])\n",
        "        min_samples = random.choice(space['min_samples'])\n",
        "        \n",
        "        clusters = generate_clusters(embeddings, \n",
        "                                     n_neighbors = n_neighbors, \n",
        "                                     n_components = n_components, \n",
        "                                     min_cluster_size = min_cluster_size,\n",
        "                                     min_samples = min_samples ,\n",
        "                                     random_state = 42)\n",
        "    \n",
        "        label_count, cost = score_clusters(clusters, prob_threshold = 0.05)\n",
        "                \n",
        "        results.append([i, n_neighbors, n_components, min_cluster_size, min_samples,\n",
        "                        label_count, cost])\n",
        "    \n",
        "    result_df = pd.DataFrame(results, columns=['run_id', 'n_neighbors', 'n_components', \n",
        "                                               'min_cluster_size', 'min_samples', 'label_count', 'cost'])\n",
        "    \n",
        "    return result_df.sort_values(by='cost')"
      ],
      "metadata": {
        "id": "rqt9sGd101hx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def objective(params, embeddings, label_lower, label_upper):\n",
        "    \"\"\"\n",
        "    Objective function for hyperopt to minimize, which incorporates constraints\n",
        "    on the number of clusters we want to identify\n",
        "    \"\"\"\n",
        "    \n",
        "    clusters = generate_clusters(embeddings, \n",
        "                                 n_neighbors = params['n_neighbors'], \n",
        "                                 n_components = params['n_components'], \n",
        "                                 min_cluster_size = params['min_cluster_size'],\n",
        "                                 min_samples = params['min_samples'],\n",
        "                                 random_state = params['random_state'])\n",
        "    \n",
        "    label_count, cost = score_clusters(clusters, prob_threshold = 0.05)\n",
        "    \n",
        "    #15% penalty on the cost function if outside the desired range of groups\n",
        "    if (label_count < label_lower) | (label_count > label_upper):\n",
        "        penalty = 0.15 \n",
        "    else:\n",
        "        penalty = 0\n",
        "    \n",
        "    loss = cost + penalty\n",
        "    \n",
        "    return {'loss': loss, 'label_count': label_count, 'status': STATUS_OK}"
      ],
      "metadata": {
        "id": "BH6zo4Pt08mr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def bayesian_search(embeddings, space, label_lower, label_upper, max_evals=100):\n",
        "    \"\"\"\n",
        "    Perform bayseian search on hyperopt hyperparameter space to minimize objective function\n",
        "    \"\"\"\n",
        "    \n",
        "    trials = Trials()\n",
        "    fmin_objective = partial(objective, embeddings=embeddings, label_lower=label_lower, label_upper=label_upper)\n",
        "        \n",
        "    best = fmin(fmin_objective, \n",
        "                space = space, \n",
        "                algo=tpe.suggest,\n",
        "                max_evals=max_evals, \n",
        "                trials=trials)\n",
        "\n",
        "    best_params = space_eval(space, best)\n",
        "    print ('best:')\n",
        "    print (best_params)\n",
        "    print (f\"label count: {trials.best_trial['result']['label_count']}\")\n",
        "    \n",
        "    best_clusters = generate_clusters(embeddings, \n",
        "                                      n_neighbors = best_params['n_neighbors'], \n",
        "                                      n_components = best_params['n_components'], \n",
        "                                      min_cluster_size = best_params['min_cluster_size'],\n",
        "                                      min_samples = best_params['min_samples'],\n",
        "                                      random_state = best_params['random_state'])\n",
        "    \n",
        "    return best_params, best_clusters, trials"
      ],
      "metadata": {
        "id": "6bzRzLK40_BS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hspace = {\n",
        "    \"n_neighbors\": hp.choice(\"n_neighbors\",range(4,15)),\n",
        "    \"n_components\": hp.choice(\"n_components\", range(2,6)),\n",
        "    \"min_cluster_size\": hp.choice(\"min_cluster_size\", range(25,45)),\n",
        "    \"min_samples\": hp.choice(\"min_samples\", range(10,35)),\n",
        "    \"random_state\": 42\n",
        "}\n",
        "\n",
        "# Label_* corresponds to the assumed lower/upper limit of topics\n",
        "# max_evals refers to the iterations of randomly selected parameters\n",
        "label_lower = 70\n",
        "label_upper = 130\n",
        "max_evals = 300"
      ],
      "metadata": {
        "id": "ONej7R901Al_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "best_params, best_cluster_use, trials_use = bayesian_search(embeddings,\n",
        "                                                           space = hspace,\n",
        "                                                           label_lower = label_lower,\n",
        "                                                           label_upper = label_upper,\n",
        "                                                           max_evals = max_evals)"
      ],
      "metadata": {
        "id": "ZtZdqnOk1PVP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Evaluation:"
      ],
      "metadata": {
        "id": "1WJsFLoQje_7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Get coherence scores\n",
        "\n",
        "from bertopic import BERTopic\n",
        "import gensim.corpora as corpora\n",
        "from gensim.models.coherencemodel import CoherenceModel\n",
        "\n",
        "model_bert = BERTopic(\n",
        "    umap_model=umap_model,\n",
        "    hdbscan_model=hdbscan_model,\n",
        "    embedding_model=embedding_model,\n",
        "    vectorizer_model=vectorizer_model,\n",
        "    top_n_words=15,\n",
        "    language='english',\n",
        "    calculate_probabilities=True,\n",
        "    verbose=True,\n",
        "    diversity=0.35\n",
        ")\n",
        "\n",
        "topics, _ = model_bert.fit_transform(docs,embeddings)\n",
        "\n",
        "# Preprocess Documents\n",
        "documents = pd.DataFrame({\"Document\": docs,\n",
        "                          \"ID\": range(len(docs)),\n",
        "                          \"Topic\": topics})\n",
        "documents_per_topic = documents.groupby(['Topic'], as_index=False).agg({'Document': ' '.join})\n",
        "cleaned_docs = model_bert._preprocess_text(documents_per_topic.Document.values)\n",
        "\n",
        "# Extract vectorizer and analyzer from BERTopic\n",
        "vectorizer = model_bert.vectorizer_model\n",
        "analyzer = vectorizer.build_analyzer()\n",
        "\n",
        "# Extract features for Topic Coherence evaluation\n",
        "words = vectorizer.get_feature_names_out()\n",
        "tokens = [analyzer(doc) for doc in cleaned_docs]\n",
        "dictionary = corpora.Dictionary(tokens)\n",
        "corpus = [dictionary.doc2bow(token) for token in tokens]\n",
        "topic_words = [[words for words, _ in model_bert.get_topic(topic)] \n",
        "               for topic in range(len(set(topics))-1)]\n",
        "\n",
        "# Evaluate\n",
        "coherence_model = CoherenceModel(topics=topic_words, \n",
        "                                 texts=tokens, \n",
        "                                 corpus=corpus,\n",
        "                                 dictionary=dictionary, \n",
        "                                 coherence='c_v') # can also use 'c_uci', 'c_npmi', 'u_mass'\n",
        "\n",
        "coherence = coherence_model.get_coherence()\n",
        "\n",
        "coherence"
      ],
      "metadata": {
        "id": "ePY6uy6Sjhk3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Topic Diversity \n",
        "\n",
        "def proportion_unique_words(model_bert, topk=15):\n",
        "    \"\"\"\n",
        "    compute the proportion of unique words\n",
        "    Parameters\n",
        "    ----------\n",
        "    bert_model: fitted BERTopic model\n",
        "    topk: top k words on which the topic diversity will be computed\n",
        "    \"\"\"\n",
        "\n",
        "    topics_list = model_bert.get_topics()\n",
        "    topics = [[words for words, _ in model_bert.get_topic(topic)] \n",
        "                  for topic in range(len(set(topics_list))-1)]\n",
        "\n",
        "    if topk > len(topics[0]):\n",
        "        raise Exception('Words in topics are less than '+str(topk))\n",
        "    else:\n",
        "        unique_words = set()\n",
        "        for topic in topics:\n",
        "            unique_words = unique_words.union(set(topic[:topk]))\n",
        "        puw = len(unique_words) / (topk * len(topics))\n",
        "        return puw\n",
        "\n",
        "\n",
        "puw = proportion_unique_words(model_bert)\n",
        "\n",
        "puw"
      ],
      "metadata": {
        "id": "BeggfqPaj_-W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Topic Diversity \n",
        "\n",
        "from itertools import combinations\n",
        "\n",
        "def pairwise_jaccard_diversity(model_bert, topk=15):\n",
        "    '''\n",
        "    compute the average pairwise jaccard distance between the topics \n",
        "  \n",
        "    Parameters\n",
        "    ----------\n",
        "    bert_model: fitted BERTopic model\n",
        "    topk: top k words on which the topic diversity will be computed\n",
        "    \n",
        "    Returns\n",
        "    -------\n",
        "    pjd: average pairwise jaccard distance\n",
        "    '''\n",
        "\n",
        "    topics_list = model_bert.get_topics()\n",
        "    topics = [[words for words, _ in model_bert.get_topic(topic)] \n",
        "                  for topic in range(len(set(topics_list))-1)]\n",
        "\n",
        "    dist = 0\n",
        "    count = 0\n",
        "    for list1, list2 in combinations(topics, 2):\n",
        "        js = 1 - len(set(list1).intersection(set(list2)))/len(set(list1).union(set(list2)))\n",
        "        dist = dist + js\n",
        "        count = count + 1\n",
        "    return dist/count\n",
        "\n",
        "pjd = pairwise_jaccard_diversity(model_bert)\n",
        "\n",
        "pjd"
      ],
      "metadata": {
        "id": "jLRDHU39Rfry"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}