src/0_lsms_processing/1_check_lsms_availability.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check available LSMS data\n",
    "Apparently not every country has not a lot of LSMS. In the following code we will check how many countries do have a LSMS and which ones have at least two, so we can perform the time travel. For checking this we are using the [search API](https://microdata.worldbank.org/api-documentation/catalog/index.html#operation/CatalogSearch) provided by the World Bank. We search for the keyword (the parameter `sk`) LSMS and specify the country by using the parameter `country`. For the country we use the ISO."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm \n",
    "import pandas as pd\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url: str = \"https://microdata.worldbank.org/index.php/api/catalog/search?sk=lsms&country=\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df: pd.DataFrame = pd.read_csv(\"../../data/countries_meta/countries_code.csv\", index_col=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 49/49 [00:42<00:00,  1.16it/s]\n"
     ]
    }
   ],
   "source": [
    "time_valid: pd.DataFrame = pd.DataFrame() # dataframe with country, iso, year and url for time travel\n",
    "valid: pd.DataFrame = pd.DataFrame() # dataframe with country, iso, year and url for only one LSMS\n",
    "\n",
    "for _, country in tqdm(df.iterrows(), total=len(df)):\n",
    "    result: any = requests.get(base_url + country[\"iso\"]).json()\n",
    "    result = result[\"result\"]\n",
    "\n",
    "    if result[\"found\"] == 0:\n",
    "        continue\n",
    "    \n",
    "    count_valid: int = 0 # counter to count valid results\n",
    "    years = [] # save years\n",
    "    urls = [] # save urls\n",
    "    for res in result[\"rows\"]:\n",
    "\n",
    "        if len(res[\"nation\"].split(\",\")) > 1: # only if the country is alone in the `nation` tag, it is the desired data\n",
    "            continue\n",
    "        \n",
    "        years.append(res[\"year_end\"])\n",
    "        urls.append(res[\"url\"])\n",
    "        count_valid += 1\n",
    "    \n",
    "    if count_valid > 0: # we just need to create a dataframe when, we found something\n",
    "        tmp_df: pd.DataFrame = pd.DataFrame.from_dict({\"name\": country[\"name\"], \"iso\": country[\"iso\"], \"year\": years, \"url\": urls})\n",
    "       \n",
    "    if count_valid == 1:\n",
    "        valid = valid.append(tmp_df)\n",
    "    \n",
    "    if count_valid > 1:\n",
    "        time_valid = time_valid.append(tmp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid.to_csv(\"../../data/countries_meta/counties_lsms_valid.csv\", index=False)\n",
    "time_valid.to_csv(\"../../data/countries_meta/counties_lsms_time_valid.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}