descriptive_statistics_computation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook computes the descriptive statistics of the metadata and human annotations.\n",
    "\n",
    "* Input: 'AgoraSpeech_preprocessed.csv'\n",
    "* Output: prints\n",
    "* Actions: For the entire dataset and per individual politician, it computes:\n",
    "    1. Total number of speeches\n",
    "    2. Total number of paragraphs\n",
    "    3. Total number of words\n",
    "    4. Average paragraphs per speech\n",
    "    5. Average words per speech\n",
    "    6. Average words per paragraph\n",
    "    7. Percentage of paragraphs identified as 'criticism'\n",
    "    8. Percentage of paragraphs identified as 'political agenda'\n",
    "    9. Sum of unique topics of all speeches\n",
    "    10. Average of unique topics per speech\n",
    "    11. Average sentiment score of all speeches\n",
    "    12. Average sentiment score per speech\n",
    "    13. Average polarization score of all speeches\n",
    "    14. Average polarization score per speech\n",
    "    15. Average populism score of all speeches\n",
    "    16. Average populism score per speech\n",
    "    17. Sum of unique entities of all speeches\n",
    "    18. Average of unique entities per speech\n",
    "    19. Average of unique entities per paragraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "import pandas as pd\n",
    "warnings.filterwarnings('ignore')\n",
    "from descriptive_statistics_functions import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the preprocessed data\n",
    "data = pd.read_csv('AgoraSpeech_preprocessed.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Descriptive statistics for the entire dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statistics for the overall dataset ... \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "General statistics:\n",
      "-------------------\n",
      "total number of speeches:  171\n",
      "total number of paragraphs:  5279\n",
      "total number of words:  717718\n",
      "avg paragraphs per speech:  30.87\n",
      "avg words per speech  4197.18\n",
      "avg words per paragraph:  135.96\n",
      "Criticism vs agenda percentages:\n",
      "--------------------------------\n",
      "agenda percentage:  0.6098\n",
      "criticism percentage:  0.3902\n",
      "Topics statistics:\n",
      "------------------\n",
      "sum_of_unique_topics_of_all_speeches:  33\n",
      "avg_of_unique_topics_of_per_speech:  12.53\n",
      "Sentiment, polarization and populsim statistics:\n",
      "------------------------------------------------\n",
      "sentiment_avg:  0.03\n",
      "sentiment_avg_per_speech:  0.01\n",
      "polarization_avg:  0.16\n",
      "polarization_avg_per_speech:  0.16\n",
      "populism_avg:  0.07\n",
      "populism_avg_per_speech:  0.07\n",
      "Entities statistics:\n",
      "--------------------\n",
      "sum of unique entities of all speeches:  7763\n",
      "avg of unique entities per speech:  122.61\n",
      "avg of unique entities per paragraph:  5.66\n"
     ]
    }
   ],
   "source": [
    "print(f\"Statistics for the overall dataset ... \")\n",
    "all_statistics(data, overall=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Descriptive statistics per individual politician"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statistics for Androulakis ... \n",
      "General statistics:\n",
      "-------------------\n",
      "total number of speeches:  26\n",
      "total number of paragraphs:  883\n",
      "total number of words:  132620\n",
      "avg paragraphs per speech:  33.96\n",
      "avg words per speech  5100.77\n",
      "avg words per paragraph:  150.19\n",
      "Criticism vs agenda percentages:\n",
      "--------------------------------\n",
      "agenda percentage:  0.6025\n",
      "criticism percentage:  0.3975\n",
      "Topics statistics:\n",
      "------------------\n",
      "sum_of_unique_topics_of_all_speeches:  32\n",
      "avg_of_unique_topics_of_per_speech:  15.58\n",
      "Sentiment, polarization and populsim statistics:\n",
      "------------------------------------------------\n",
      "sentiment_avg:  0.0\n",
      "sentiment_avg_per_speech:  -0.01\n",
      "polarization_avg:  0.17\n",
      "polarization_avg_per_speech:  0.16\n",
      "populism_avg:  0.09\n",
      "populism_avg_per_speech:  0.09\n",
      "Entities statistics:\n",
      "--------------------\n",
      "sum of unique entities of all speeches:  1605\n",
      "avg of unique entities per speech:  137.92\n",
      "avg of unique entities per paragraph:  6.51\n",
      "\n",
      "\n",
      "Statistics for Koutsoumpas ... \n",
      "General statistics:\n",
      "-------------------\n",
      "total number of speeches:  27\n",
      "total number of paragraphs:  707\n",
      "total number of words:  98807\n",
      "avg paragraphs per speech:  26.19\n",
      "avg words per speech  3659.52\n",
      "avg words per paragraph:  139.76\n",
      "Criticism vs agenda percentages:\n",
      "--------------------------------\n",
      "agenda percentage:  0.5248\n",
      "criticism percentage:  0.4752\n",
      "Topics statistics:\n",
      "------------------\n",
      "sum_of_unique_topics_of_all_speeches:  31\n",
      "avg_of_unique_topics_of_per_speech:  9.85\n",
      "Sentiment, polarization and populsim statistics:\n",
      "------------------------------------------------\n",
      "sentiment_avg:  -0.25\n",
      "sentiment_avg_per_speech:  -0.25\n",
      "polarization_avg:  0.26\n",
      "polarization_avg_per_speech:  0.26\n",
      "populism_avg:  0.11\n",
      "populism_avg_per_speech:  0.12\n",
      "Entities statistics:\n",
      "--------------------\n",
      "sum of unique entities of all speeches:  1531\n",
      "avg of unique entities per speech:  104.96\n",
      "avg of unique entities per paragraph:  5.97\n",
      "\n",
      "\n",
      "Statistics for Mitsotakis ... \n",
      "General statistics:\n",
      "-------------------\n",
      "total number of speeches:  45\n",
      "total number of paragraphs:  1385\n",
      "total number of words:  147709\n",
      "avg paragraphs per speech:  30.78\n",
      "avg words per speech  3282.42\n",
      "avg words per paragraph:  106.65\n",
      "Criticism vs agenda percentages:\n",
      "--------------------------------\n",
      "agenda percentage:  0.8491\n",
      "criticism percentage:  0.1509\n",
      "Topics statistics:\n",
      "------------------\n",
      "sum_of_unique_topics_of_all_speeches:  31\n",
      "avg_of_unique_topics_of_per_speech:  12.47\n",
      "Sentiment, polarization and populsim statistics:\n",
      "------------------------------------------------\n",
      "sentiment_avg:  0.46\n",
      "sentiment_avg_per_speech:  0.49\n",
      "polarization_avg:  0.07\n",
      "polarization_avg_per_speech:  0.07\n",
      "populism_avg:  0.02\n",
      "populism_avg_per_speech:  0.02\n",
      "Entities statistics:\n",
      "--------------------\n",
      "sum of unique entities of all speeches:  2050\n",
      "avg of unique entities per speech:  99.96\n",
      "avg of unique entities per paragraph:  4.51\n",
      "\n",
      "\n",
      "Statistics for Tsipras ... \n",
      "General statistics:\n",
      "-------------------\n",
      "total number of speeches:  44\n",
      "total number of paragraphs:  1601\n",
      "total number of words:  185139\n",
      "avg paragraphs per speech:  36.39\n",
      "avg words per speech  4207.7\n",
      "avg words per paragraph:  115.64\n",
      "Criticism vs agenda percentages:\n",
      "--------------------------------\n",
      "agenda percentage:  0.5353\n",
      "criticism percentage:  0.4647\n",
      "Topics statistics:\n",
      "------------------\n",
      "sum_of_unique_topics_of_all_speeches:  31\n",
      "avg_of_unique_topics_of_per_speech:  13.89\n",
      "Sentiment, polarization and populsim statistics:\n",
      "------------------------------------------------\n",
      "sentiment_avg:  -0.05\n",
      "sentiment_avg_per_speech:  -0.06\n",
      "polarization_avg:  0.16\n",
      "polarization_avg_per_speech:  0.16\n",
      "populism_avg:  0.09\n",
      "populism_avg_per_speech:  0.09\n",
      "Entities statistics:\n",
      "--------------------\n",
      "sum of unique entities of all speeches:  2597\n",
      "avg of unique entities per speech:  129.84\n",
      "avg of unique entities per paragraph:  4.99\n",
      "\n",
      "\n",
      "Statistics for Varoufakis ... \n",
      "General statistics:\n",
      "-------------------\n",
      "total number of speeches:  18\n",
      "total number of paragraphs:  525\n",
      "total number of words:  124716\n",
      "avg paragraphs per speech:  29.17\n",
      "avg words per speech  6928.67\n",
      "avg words per paragraph:  237.55\n",
      "Criticism vs agenda percentages:\n",
      "--------------------------------\n",
      "agenda percentage:  0.5086\n",
      "criticism percentage:  0.4914\n",
      "Topics statistics:\n",
      "------------------\n",
      "sum_of_unique_topics_of_all_speeches:  28\n",
      "avg_of_unique_topics_of_per_speech:  11.0\n",
      "Sentiment, polarization and populsim statistics:\n",
      "------------------------------------------------\n",
      "sentiment_avg:  -0.27\n",
      "sentiment_avg_per_speech:  -0.29\n",
      "polarization_avg:  0.18\n",
      "polarization_avg_per_speech:  0.18\n",
      "populism_avg:  0.09\n",
      "populism_avg_per_speech:  0.09\n",
      "Entities statistics:\n",
      "--------------------\n",
      "sum of unique entities of all speeches:  1911\n",
      "avg of unique entities per speech:  192.67\n",
      "avg of unique entities per paragraph:  8.53\n",
      "\n",
      "\n",
      "Statistics for Velopoulos ... \n",
      "General statistics:\n",
      "-------------------\n",
      "total number of speeches:  11\n",
      "total number of paragraphs:  178\n",
      "total number of words:  28727\n",
      "avg paragraphs per speech:  16.18\n",
      "avg words per speech  2611.55\n",
      "avg words per paragraph:  161.39\n",
      "Criticism vs agenda percentages:\n",
      "--------------------------------\n",
      "agenda percentage:  0.7135\n",
      "criticism percentage:  0.2865\n",
      "Topics statistics:\n",
      "------------------\n",
      "sum_of_unique_topics_of_all_speeches:  27\n",
      "avg_of_unique_topics_of_per_speech:  9.18\n",
      "Sentiment, polarization and populsim statistics:\n",
      "------------------------------------------------\n",
      "sentiment_avg:  -0.44\n",
      "sentiment_avg_per_speech:  -0.44\n",
      "polarization_avg:  0.2\n",
      "polarization_avg_per_speech:  0.21\n",
      "populism_avg:  0.05\n",
      "populism_avg_per_speech:  0.05\n",
      "Entities statistics:\n",
      "--------------------\n",
      "sum of unique entities of all speeches:  600\n",
      "avg of unique entities per speech:  78.91\n",
      "avg of unique entities per paragraph:  6.74\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for politician in data['politician'].unique():\n",
    "    politician_df = data[data['politician'] == politician]\n",
    "    print(f\"Statistics for {politician} ... \")\n",
    "    all_statistics(politician_df)\n",
    "    print(\"\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}