pgm_features_competition_codebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfe253f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import re\n",
    "import pandas as pd\n",
    "import json\n",
    "import markdown\n",
    "import io\n",
    "\n",
    "import os\n",
    "home = os.path.expanduser(\"~\")\n",
    "\n",
    "print('OS path is set to:', home)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "969faa88",
   "metadata": {},
   "outputs": [],
   "source": [
    "desktop = home+'/Desktop' # (Toggle on to save to desktop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1437f1ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If you want to create a codebook based on a subset of data from a file, read that file as a dataframe from your working directory\n",
    "df = pd.read_parquet('pgm_features_to_oct2019.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1e6a879",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Change to the name you want to use to describe created codebook \n",
    "codebook = 'pgm_features_competition'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed25a262",
   "metadata": {},
   "outputs": [],
   "source": [
    "#get all the column names from your data\n",
    "list(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97e406ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create empty df\n",
    "df_new = pd.DataFrame()\n",
    "#get all variable names as rows\n",
    "df_new['variable_name']= df.columns.values.tolist() \n",
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "002db020",
   "metadata": {},
   "outputs": [],
   "source": [
    "#These are common pgm transforms used in VIEWS, change as needed. \n",
    "def give_me_transforms(df_new):\n",
    "    headers = []\n",
    "    if \"splag\" in df_new: headers.append('spatial lag')\n",
    "    if \"decay\" in df_new: headers.append('decay function')\n",
    "    if \"t48\" in df_new: headers.append('temporal lag of 48 months')\n",
    "    if \"tlag\" in df_new: headers.append('temporal lag of X months')\n",
    "    if \"tsum_24\" in df_new: headers.append('24-month moving sum')\n",
    "    if \"ln\" in df_new: headers.append('natural log')\n",
    "    if \"treelag\" in df_new: headers.append('Spatial lag without kernel - an approximate weighted sum over the whole pg grid is computed at every grid cell (ignoring the target cell itself).')\n",
    "    if \"mov_avg\" in df_new: headers.append('moving average')\n",
    "    if \"mov_sum\" in df_new: headers.append('moving sum')\n",
    "    if \"sptime_dist\" in df_new: headers.append('the spacetime distance to the nearest k present or past events, where an event is any (pg cell,timestep) which has a non-zero value of the input feature')\n",
    "    if \"gte\" in df_new: headers.append('Returns 1 if feature values are greater than or equal to the supplied parameter, zero otherwise.')\n",
    "    if \"greq\" in df_new: headers.append('Returns 1 if feature values are greater than or equal to the supplied parameter, zero otherwise.')\n",
    "    return headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daeddb92",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate the transform_description column based on the variable_name column\n",
    "df_new['transform_description'] = df_new['variable_name'].apply(give_me_transforms)\n",
    "\n",
    "# Combine the list of header strings for each column into a single string\n",
    "df_new['transform_description'] = df_new['transform_description'].apply(lambda x: \", \".join(x))\n",
    "\n",
    "# Print the updated DataFrame\n",
    "print(df_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf566161",
   "metadata": {},
   "outputs": [],
   "source": [
    "#match the variable names with the descriptions\n",
    "## you can change it to match the descriptions, so \"if 'wdi' in df_name: header = 'world development index'\"\n",
    "## you can repeat it for as many columns you need \n",
    "header= ''\n",
    "def give_me_description(df_new):\n",
    "    global header\n",
    "    if \"ged\" in df_new: header = \"UCDP Georeferenced Events Dataset (UCDP GED)\"\n",
    "    if \"wdi\" in df_new: header = \"World Development Index\"\n",
    "    if \"pop_gpw_sum\" in df_new: header = 'Center for International Earth Science Information Network (CIESIN) and Centro Internacional de Agricultura Tropical (CIAT) (2005). Gridded Population of the World, Version 3'\n",
    "    if \"nlights_calib_mean\" in df_new: header = \"Elvidge, Christopher D., Feng-Chi Hsu, Kimberly E. Baugh and Tilottama Ghosh (2014). “National Trends in Satellite Observed Lighting: 1992-2012.” Global Urban Monitoring and Assessment Through Earth Observation. Ed. Qihao Weng. CRC Press; PRIO- GRID version 2.0\"\n",
    "    if \"imr_mean\" in df_new: header = \"Storeygard, Adam; Deborah Balk, Marc Levy & Glenn Deane (2008) The global distribution of infant mortality: A subnational spatial view. Population, Space and Place, 14(3):209-229; Center for International Earth Science Information Network - CIESIN - Columbia University. 2005. Poverty Mapping Project: Global Subnational Infant Mortality Rates. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC).\"\n",
    "    if \"urban_ih\" in df_new: header = \"Meiyappan, Prasanth and Atul K. Jain (2012). Three distinct global estimates of historical land-cover change and land-use conversions for over 200 years. Frontiers of Earth Science, 6(2), 122-139.\"\n",
    "    if \"count_moder_drought_prev10\" in df_new: header = \"Computation from Vesco, P. (2021). A Climate of War or Peace? The Effect of Droughts on Conflict Dynamics.\"\n",
    "    if \"cropprop\" in df_new: header = \"Mapspam (International Food Policy Research Institute 2019).\"\n",
    "    if \"growseasdummy\" in df_new: header = \"Portmann, F.T., S. Siebert, and P. Döll. 2010. MIRCA2000 - Global monthly irrigated and rainfed crop area around the year 2000: a new high-resolution data set for agricultural and hydrological modeling. Global Biogeochemical Cycles, Vol. 24\"\n",
    "    if \"spei1\" in df_new: header = \"Computation from SPEI Drought Monitor (Vicente-Serrano et al., 2010).\"\n",
    "    if \"tlag1_dr\" in df_new: header = \"Standardized Precipitation Evapotranspiration Index (Vicente-Serrano et al., 2010).\"\n",
    "    if \"tlag_12_\" in df_new: header = \"Mapspam (International Food Policy Research Institute 2019).\"\n",
    "    if \"excluded\" in df_new: header = \"Vogt, Manuel, Nils-Christian Bormann, Seraina Rüegger, Lars-Erik Cederman, Philipp Hunziker, and Luc Girardin. 2015. Integrating Data on Ethnicity, Geography, and Conflict: The Ethnic Power Relations Dataset Family. Journal of Conflict Resolution, 59(7), 1327-1342.\"\n",
    "    if \"pgd_ttime_mean\" in df_new: header = \"Uchida, Hirotsugu and Nelson, Andrew (2009). Agglomeration Index: Towards a New Measure of Urban Concentration. Background paper for the World Bank’s World Development Report 2009\"\n",
    "    if \"imr_mean\" in df_new: header = \"Storeygard, Adam; Deborah Balk, Marc Levy & Glenn Deane (2008) The global distribution of infant mortality: A subnational spatial view. Population, Space and Place, 14(3):209-229; Center for International Earth Science Information Network - CIESIN - Columbia University. 2005. Poverty Mapping Project: Global Subnational Infant Mortality Rates. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC).\"\n",
    "    if \"urban_ih\" in df_new: header = \"Meiyappan, Prasanth and Atul K. Jain (2012). Three distinct global estimates of historical land-cover change and land-use conversions for over 200 years. Frontiers of Earth Science, 6(2), 122-139.\"\n",
    "    if \"mountains_mean\" in df_new: header = \"Blyth, Simon, Brian Groombridge, Igor Lysenko, Lera Miles, and Adrian Newton (2002). Mountain Watch: environmental change & sustainable development in moun- tains. UNEP-WCMC Biodiversity Series 12. ISBN: 1-899628-20-7\"\n",
    "    if \"dist_diamsec\" in df_new: header = \"PRIO- GRID version 2.0, please see Tollefsen, Andreas Forø; Håvard Strand & Halvard Buhaug (2012) PRIO-GRID: A unified spatial data structure. Journal of Peace Research, 49(2): 363-374.\"\n",
    "    if \"dist_petroleum\" in df_new: header = \"PRIO- GRID version 2.0, please see Tollefsen, Andreas Forø; Håvard Strand & Halvard Buhaug (2012) PRIO-GRID: A unified spatial data structure. Journal of Peace Research, 49(2): 363-374.\"\n",
    "    if \"agri_ih\" in df_new: header = \"Meiyappan, Prasanth and Atul K. Jain (2012). Three distinct global estimates of historical land-cover change and land-use conversions for over 200 years. Frontiers of Earth Science, 6(2), 122-139.\"\n",
    "    if \"barren_ih\" in df_new: header = \"Meiyappan, Prasanth and Atul K. Jain (2012). Three distinct global estimates of historical land-cover change and land-use conversions for over 200 years. Frontiers of Earth Science, 6(2), 122-139.\"\n",
    "    if \"forest_ih\" in df_new: header = \"Meiyappan, Prasanth and Atul K. Jain (2012). Three distinct global estimates of historical land-cover change and land-use conversions for over 200 years. Frontiers of Earth Science, 6(2), 122-139.\"\n",
    "    if \"pasture_ih\" in df_new: header = \"Meiyappan, Prasanth and Atul K. Jain (2012). Three distinct global estimates of historical land-cover change and land-use conversions for over 200 years. Frontiers of Earth Science, 6(2), 122-139.\"\n",
    "    if \"savanna_ih\" in df_new: header = \"PRIO- GRID version 2.0, please see Tollefsen, Andreas Forø; Håvard Strand & Halvard Buhaug (2012) PRIO-GRID: A unified spatial data structure. Journal of Peace Research, 49(2): 363-374.\"\n",
    "    if \"shrub_ih\" in df_new: header = \"PRIO- GRID version 2.0, please see Tollefsen, Andreas Forø; Håvard Strand & Halvard Buhaug (2012) PRIO-GRID: A unified spatial data structure. Journal of Peace Research, 49(2): 363-374.\"\n",
    "    if \"urban_ih\" in df_new: header = \"Meiyappan, Prasanth and Atul K. Jain (2012). Three distinct global estimates of historical land-cover change and land-use conversions for over 200 years. Frontiers of Earth Science, 6(2), 122-139.\"\n",
    "    if \"ttime_mean\" in df_new: header = \"Uchida, Hirotsugu and Nelson, Andrew (2009). Agglomeration Index: Towards a New Measure of Urban Concentration. Background paper for the World Bank’s World Development Report 2009; PRIO- GRID version 2.0.\"\n",
    "    if \"gcp_mer\" in df_new: header = \"Nordhaus, William D. (2006) Geography and macroeconomics: New data and new findings. Proceedings of the National Academy of Sciences of the USA, 103(10): 3510- 3517; PRIO- GRID version 2.0\"\n",
    "    if \"bdist3\" in df_new: header = \"Weidmann, Nils B., Doreen Kuse & Kristian Skrede Gleditsch (2010) The geography of the international system: The CShapes Dataset. International Interactions, 36(1): 86-106; PRIO- GRID version 2.0\"\n",
    "    if \"capdist\" in df_new: header = \"Weidmann, Nils B., Doreen Kuse & Kristian Skrede Gleditsch (2010) The geography of the international system: The CShapes Dataset. International Interactions, 36(1): 86-106; PRIO- GRID version 2.0\"\n",
    "    return header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6df3ac83",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new['source_description'] = df_new['variable_name'].apply(give_me_description)\n",
    "print(df_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "270bb191",
   "metadata": {},
   "outputs": [],
   "source": [
    "#match the variable names with the descriptions\n",
    "## you can change it to match the descriptions, so \"if 'wdi' in df_name: header = 'world development index'\"\n",
    "## you can repeat it for as many columns you need \n",
    "#These columns can also be merged from the VIEWS pgm codebook \n",
    "header= ''\n",
    "def give_me_shortdescription(df_new):\n",
    "    global header\n",
    "    if \"sb\" in df_new: header = 'The best (most likely) estimate of the total number of battle-related deaths (BRDs) from state-based conflict as per the UCDP definition.'\n",
    "    if \"ns\" in df_new: header = \"The best (most likely) estimate of the total number of fatalities from non-state conflict as per the UCDP definition.\"\n",
    "    if \"os\" in df_new: header = \"The best (most likely) estimate of the total number of fatalities from one-sided conflict as per the UCDP definition.\"\n",
    "    if \"pop_gpw_sum\" in df_new: header = \"The sum of pixel values (number of persons) within the grid cell.\"\n",
    "    if \"nlights_calib_mean\" in df_new: header = \"Average nighttime light emission with values standardized to be between 0 and 1, where 1 is the highest observed value in the time-series, and 0 is the lowest\"\n",
    "    if \"imr_mean\" in df_new: header = \"The average infant mortality rate within the grid cell.\"\n",
    "    if \"urban_ih\" in df_new: header = \"The percentage area of the cell covered by urban area.\"\n",
    "    if \"urban_ih\" in df_new: header = \"The percentage area of the cell covered by urban area.\"   \n",
    "    if \"count_moder_drought_prev10\" in df_new: header = \"Count of the months experiencing a moderate drought in the previous 10 years.\"\n",
    "    if \"cropprop\" in df_new: header = \"Proportion of months in a year for which the growing season is ongoing, lagged by 12 months.\"\n",
    "    if \"growseasdummy\" in df_new: header = \"Binary variable indicating if the growing season is ongoing for each month and grid-cell.\"\n",
    "    if \"spei1_gs_prev10\" in df_new: header = \"Difference between current value of agricultural SPEI during the growing season and its average value for the ten previous years.\"\n",
    "    if \"spei1_gs_prev10_anom\" in df_new: header = \"Difference between current value of agricultural SPEI during the growing season and its average value for the ten previous years.\"\n",
    "    if \"spei1_gsm_cv_anom\" in df_new: header = \"Difference between the temporal coefficient of variation of spei1 gsm along the current year, and the average variation of spei1 gsm over the period 1990-2010.\"\n",
    "    if \"spei1_gsm_detrend\" in df_new: header = \"SPEI value for the months in which the growing season is ongoing. Detrended.\"\n",
    "    if \"spei1gsy_lowermedian_count\" in df_new: header = \"Consecutive number of years in which drought exceeds median value for sample.\"\n",
    "    if \"spei_48_detrend\" in df_new: header = \"Long-term drought (4-year average SPEI) detrended.\"    \n",
    "    if \"tlag1_dr_mod_gs\" in df_new: header = \"Occurrence of a modest drought during the growing season (SPEI value lower than -0.5), lagged by one month.\"\n",
    "    if \"tlag1_dr_moder_gs\" in df_new: header = \"Occurrence of a moderate drought during the growing season (SPEI value lower than -1.5), lagged by one month.\"\n",
    "    if \"tlag1_dr_sev_gs\" in df_new: header = \"Occurrence of a severe drought during the growing season (SPEI value lower than -2), lagged by one month.\"\n",
    "    if \"tlag1_spei1_gsm\" in df_new: header = \"Agricultural drought, proxied by the SPEI value during the growing season months. Lagged by 1 month.\"\n",
    "    if \"tlag_12_crop_sum\" in df_new: header = \"Harvest quantity of the main crops, capturing harvest failure. Lagged by 12 months.\"\n",
    "    if \"tlag_12_harvarea_maincrops\" in df_new: header = \"Harvested area of the main crops in the priogrid-cell. Lagged by 12 months.\"\n",
    "    if \"tlag_12_irr_maincrops\" in df_new: header = \"Dummy indicating whether the main crops are irrigated, lagged by 12 month.\"\n",
    "    if \"tlag_12_rainf_maincrops\" in df_new: header = \"Dummy indicating whether the main crops are rainfed, lagge dby 12 months.\"\n",
    "    if \"excluded\" in df_new: header = \"Number of excluded groups (discriminated or powerless) settled in the grid cell for the given year.\"\n",
    "    if \"ttime_mean\" in df_new: header = \"The average travel time in minutes to the nearest major city within each cell.\"\n",
    "    if \"wdi_nv_agr_totl_kd\" in df_new: header = \"Value added from agriculture in constant 2015 U.S. dollars.\"\n",
    "    if \"imr_mean\" in df_new: header = \"The average infant mortality rate within the grid cell.\"\n",
    "    if \"urban_ih\" in df_new: header = \"The percentage area of the cell covered by urban area.\"\n",
    "    if \"mountains_mean\" in df_new: header = \"The proportion of mountainous terrain within the cell based on elevation, slope and local elevation range.\"\n",
    "    if \"dist_diamsec\" in df_new: header = \"Distance to nearest primary diamond resource\"\n",
    "    if \"dist_petroleum\" in df_new: header = \"Distance to nearest petroleum resource\"\n",
    "    if \"agri_ih\" in df_new: header = \"The percentage area of the cell covered by agricultural area.\"\n",
    "    if \"barren_ih\" in df_new: header = \"The percentage area of the cell covered by barren area.\"\n",
    "    if \"forest_ih\" in df_new: header = \"The percentage area of the cell covered by forest area.\"\n",
    "    if \"pasture_ih\" in df_new: header = \"The percentage area of the cell covered by pasture area.\"\n",
    "    if \"savanna_ih\" in df_new: header = \"The percentage area of the cell covered by grasslands.\"\n",
    "    if \"shrub_ih\" in df_new: header = \"The percentage area of the cell covered by shrublands.\"\n",
    "    if \"urban_ih\" in df_new: header = \"The percentage area of the cell covered by urban area.\"\n",
    "    if \"ttime_mean\" in df_new: header = \"The average travel time in minutes to the nearest major city within each cell.\"\n",
    "    if \"gcp_mer\" in df_new: header = \"Indicates the gross cell product, measured in USD.\"\n",
    "    if \"bdist3\" in df_new: header = \"The spherical distance (in kilometer) from the cell centroid to the territorial outline of the country the cell belongs to.\"\n",
    "    if \"capdist\" in df_new: header = \"The spherical distance in kilometers from the cell centroid to the national capital city.\"\n",
    "    return header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cca8b190",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new['short_description'] = df_new['variable_name'].apply(give_me_shortdescription)\n",
    "print(df_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f722c5c8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "552d3263",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_new.to_markdown())\n",
    "\n",
    "#Copy paste this markdown table into a new notebook, switch to markdown language and run one cell with the pasted table. \n",
    "#Then click file and download the entire notebook as LaTex - you can then use this LaTex code to produce a nice PDF in overleaf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6635436",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_file = df_new.to_json(orient=\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0d21f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new.to_json(desktop + f'/{codebook}.json')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}