graph_matching/monty_world_visualizations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "999a21af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import pprint\n",
    "from scipy.spatial.transform import Rotation\n",
    "\n",
    "from tbp.monty.frameworks.utils.logging_utils import (load_stats, \n",
    "                                                        check_rotation_accuracy,\n",
    "                                                        deserialize_json_chunks)\n",
    "from tbp.monty.frameworks.utils.plot_utils import (\n",
    "    plot_graph, \n",
    "    plot_feature_matching_animation,\n",
    "    show_one_step,\n",
    "    plot_evidence_at_step,\n",
    "    plot_sample_animation,\n",
    "    PolicyPlot,\n",
    "    plot_learned_graph,\n",
    "    plot_hotspots,\n",
    "plot_rotation_stat_animation,\n",
    "plot_detection_stat_animation,)\n",
    "\n",
    "from tbp.monty.frameworks.utils.transform_utils import numpy_to_scipy_quat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d386e59",
   "metadata": {},
   "outputs": [],
   "source": [
    "episode_num = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20fa3f40",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8516b48e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify pre-training; determines point-cloud models that are visualized\n",
    "# General paths:\n",
    "pretrain_path = os.path.expanduser(\"~/tbp/results/monty/pretrained_models/\")\n",
    "log_path = os.path.expanduser(\"~/tbp/results/monty/projects/\")\n",
    "\n",
    "# Specific experiment paths:\n",
    "pretrained_dict = pretrain_path + \"pretrained_ycb_v4/touch_1lm_numenta_lab_obj/pretrained/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "976d01cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify whether curvature-informed or not\n",
    "exp_path = log_path + \"evidence_eval_runs/logs/base_config_monty_world/\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34dab620",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Load some detailed stats\n",
    "# train_stats, eval_stats, _, lm_models = load_stats(exp_path,\n",
    "#                                                                 load_train=False,\n",
    "#                                                                 load_eval=True,\n",
    "#                                                                 load_detailed=False,\n",
    "#                                                                 pretrained_dict=pretrained_dict,\n",
    "#                                                                )\n",
    "\n",
    "# # Load just a single episode from detailed stats\n",
    "# det_path = os.path.join(exp_path, \"detailed_run_stats.json\")\n",
    "# detailed_stats = deserialize_json_chunks(json_file=det_path, episodes=[episode_num])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96e75b08",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load all detailed stats\n",
    "train_stats, eval_stats, detailed_stats, lm_models = load_stats(exp_path,\n",
    "                                                                load_train=False,\n",
    "                                                                load_eval=True,\n",
    "                                                                load_detailed=True,\n",
    "                                                                pretrained_dict=pretrained_dict,\n",
    "                                                               )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78ca52f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d453d172",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for key in detailed_stats.keys():\n",
    "#     print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "718bf043",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # All keys for a particular episode\n",
    "# for key in detailed_stats[str(episode_num)].keys():\n",
    "#     print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d22abe58",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Motor system keys\n",
    "# for key in detailed_stats[str(episode_num)][\"motor_system\"].keys():\n",
    "#     print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbe944d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(detailed_stats[str(episode_num)][\"motor_system\"][\"action_details\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a24a391",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(len(detailed_stats[str(episode_num)][\"motor_system\"][\"action_sequence\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2a9877e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print((detailed_stats[str(episode_num)][\"motor_system\"][\"action_sequence\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba719336",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # SM Keys\n",
    "# for key in detailed_stats[str(episode_num)][\"SM_1\"].keys():  # All keys for a particular episode\n",
    "#     print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47aed304",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pprint.pprint(detailed_stats[str(episode_num)][\"SM_0\"][\"processed_observations\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9da4157c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(np.shape(detailed_stats[str(episode_num)][\"SM_0\"][\"processed_observations\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f2e21f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(detailed_stats[str(episode_num)][\"LM_0\"][\"lm_processed_steps\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c2f4b05",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # SM Keys\n",
    "# for item in detailed_stats[str(episode_num)][\"SM_0\"][\"raw_observations\"][100]:  # All keys for a particular episode\n",
    "#     print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bb185b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# step_num_temp=100\n",
    "# print(detailed_stats[str(episode_num)][\"SM_0\"][\"raw_observations\"][step_num_temp][\"rgba\"])\n",
    "# print(len(detailed_stats[str(episode_num)][\"SM_0\"][\"raw_observations\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0101a39",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Visualize available keys in LM\n",
    "# for key in detailed_stats[\"1\"][\"LM_0\"].keys():\n",
    "#     print(key)\n",
    "    \n",
    "# # Locations : these are the locations of the sensor module *taking into account depth*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bab4a312",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Visualize available keys\n",
    "# for key in lm_models[\"pretrained\"][0][\"mug\"]:\n",
    "#     print(key)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f88b28b3",
   "metadata": {},
   "source": [
    "### Visualize Object-Classification Biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a09b707d",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_stats = pd.read_csv(os.path.expanduser(\"~/tbp/results/monty/projects/monty_world/trained_on_full_ycb/fullycb_trained_config_monty_world_full_hsv.csv\"))\n",
    "\n",
    "all_objects = [\"numenta_mug\", \"red_mug\"]\n",
    "\n",
    "results_dic = {}\n",
    "for obj in all_objects:\n",
    "    results_dic[obj] = []\n",
    "\n",
    "num_episodes = eval_stats.shape[0]\n",
    "for episode in range(num_episodes):\n",
    "    results_dic[eval_stats.iloc[episode]['target_object']].append(eval_stats.iloc[episode]['most_likely_object'])\n",
    "\n",
    "#pprint.pprint(results_dic)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08d8f92d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need to mask by target-object\n",
    "category_results = eval_stats['most_likely_object'].value_counts()\n",
    "print(category_results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f0c294a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d1434c44",
   "metadata": {},
   "source": [
    "### Analayze Learned Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b80572ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lm_model_stats(\n",
    "    lm_models,\n",
    "    episode,\n",
    "    object_id,\n",
    "    lm_index=0,\n",
    "):\n",
    "    \"\"\"\n",
    "    Get some basic stats about e.g. how many points each LM has in its object graph\n",
    "    \"\"\"\n",
    "\n",
    "    lm = \"LM_\" + str(lm_index)\n",
    "\n",
    "    # Use point-cloud model of ground-truth object that is in the evironment\n",
    "    # This is based on the *LM's model*, but always getting the ground-truth object,\n",
    "    # i.e. regardless of whether the LM is successfully recognizing the object or not\n",
    "    # Thus we can see if e.g. there is a difference in exploration depending on how\n",
    "    # well known areas on the model are\n",
    "    learned_model_cloud = lm_models[\"pretrained\"][lm_index][object_id].pos\n",
    "    \n",
    "    return len(learned_model_cloud)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30dc3498",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "# for episode_iter in range(0,2):\n",
    "#     num_points = get_lm_model_stats(lm_models, episode=episode_iter,\n",
    "#                              object_id=detailed_stats[str(episode_iter)][\"target\"][\"target_object\"])\n",
    "#     print(\"\\nObject: \" + str(detailed_stats[str(episode_iter)][\"target\"][\"target_object\"]))\n",
    "#     print(num_points)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b35aa5c5",
   "metadata": {},
   "source": [
    "### Plot learned graphs (no policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "005a3560",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Gather \"hot-spot\" data\n",
    "\n",
    "# for current_episode in range(3):\n",
    "#     # Mask based on successful jumps; in principle don't need to, but these will be more\n",
    "#     # interesting, because these are the locations we aim for when our pose estimates tend \n",
    "#     # to (at least presumably, although not by definition) be good\n",
    "#     # Need to mask based on what the MLH object was at the time (this information is available\n",
    "#     # already in the LM data)\n",
    "#     # --> ?need to transform based on the estimated pose at the time --> no shouldn't\n",
    "#     # need to because the target location will simply be in the reference frame of the object\n",
    "#     print(detailed_stats[str(current_episode)][\"motor_system\"][\"action_details\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "858f3da6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "view_dic = dict(\n",
    "    mug=[-45,45],\n",
    "    spoon=[-45,90],\n",
    "    numenta_mug=[-45,25],\n",
    "    red_mug=[180,-45],\n",
    ")\n",
    "\n",
    "episode_num=1 # 0 is mug, 6 is spoon\n",
    "target_id=detailed_stats[str(episode_num)][\"target\"][\"target_object\"]\n",
    "\n",
    "# Plot just the learned graph\n",
    "sns.set(font_scale = 1.0)\n",
    "plot_learned_graph(detailed_stats, \n",
    "                    lm_models,  \n",
    "                    episode=episode_num,\n",
    "                    view=view_dic[target_id],\n",
    "                    object_id=target_id,\n",
    "                    save_fig=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e823d9f",
   "metadata": {},
   "source": [
    "### Plot hot-spot areas visited by top-down policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf4e8803",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "view_dic = dict(\n",
    "    mug=[25,-65],\n",
    "    spoon=[-45,90],\n",
    "    fork=[-45,110],\n",
    ")\n",
    "\n",
    "episode_num=0\n",
    "target_id=detailed_stats[str(episode_num)][\"target\"][\"target_object\"]\n",
    "\n",
    "# Plot just the learned graph\n",
    "sns.set(font_scale = 1.0)\n",
    "plot_hotspots(detailed_stats, \n",
    "                lm_models,  \n",
    "                episode_range=2,\n",
    "                view=view_dic[target_id],\n",
    "                object_id=target_id,)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "282a1211",
   "metadata": {},
   "source": [
    "### Plot policy *animation*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddd2eb36",
   "metadata": {},
   "outputs": [],
   "source": [
    "view_dic = dict(\n",
    "    mug = [35,25],  # [45,-55]\n",
    "    spoon = [45,-75],\n",
    "    golf_ball = [45, 45],\n",
    ")\n",
    "zoom_dic = dict(\n",
    "    mug = 1.35,\n",
    "    spoon = 1.35,\n",
    "    golf_ball = 0.75,\n",
    ")\n",
    "\n",
    "episode_num = 0\n",
    "\n",
    "plot_policy_animation(detailed_stats, \n",
    "                                lm_models,  \n",
    "                                episode = episode_num,\n",
    "                                step=-1,\n",
    "                                agent_type=\"vision\",\n",
    "                                jumps_used=True,\n",
    "                                object_id=detailed_stats[str(episode_num)][\"target\"][\"target_object\"],\n",
    "                                view=view_dic[detailed_stats[str(episode_num)][\"target\"][\"target_object\"]],\n",
    "                                zoom=zoom_dic[detailed_stats[str(episode_num)][\"target\"][\"target_object\"]],\n",
    "                                extra_vis=\"sensor_pose\",  # sensor_pose or lm_processed\n",
    "                                #agent_step=40,\n",
    "                        )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7d0f269",
   "metadata": {},
   "source": [
    "### Plot graph with policy, and *agent movements*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80bea85f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Good views for mug basic policy:\n",
    "[25,-150]\n",
    "[0,0]\n",
    "\"\"\"\n",
    "\n",
    "sns.set(font_scale = 1.0)\n",
    "episode_num=0\n",
    "temp_step=-1\n",
    "plot_policy_across_model(detailed_stats, \n",
    "                                lm_models,  \n",
    "                                episode = episode_num,\n",
    "                                step=30,\n",
    "                                object_id=detailed_stats[str(episode_num)][\"target\"][\"target_object\"],\n",
    "                                #view=[45,-80],\n",
    "                                #zoom=1.5,\n",
    "                                #extra_vis=\"sensor_pose\",\n",
    "                                #agent_step=40,\n",
    "                        )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49f0d057",
   "metadata": {},
   "source": [
    "### Camera observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68df5113",
   "metadata": {},
   "outputs": [],
   "source": [
    "episode_num = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de9e5e6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(detailed_stats[str(episode_num)][\"SM_0\"][\"raw_observations\"][step_num_temp][\"rgba\"])\n",
    "# print(len(detailed_stats[str(episode_num)][\"SM_0\"][\"raw_observations\"]))\n",
    "\n",
    "viz_obs = []\n",
    "patch_obs = []\n",
    "\n",
    "# assert (len(detailed_stats[str(episode_num)][\"SM_0\"][\"raw_observations\"]) == len(detailed_stats[str(episode_num)][\"SM_1\"][\"raw_observations\"]), \"Different number of obs\")\n",
    "\n",
    "for ii in range(len(detailed_stats[str(episode_num)][\"SM_0\"][\"raw_observations\"])):\n",
    "    \n",
    "    viz_obs.append(detailed_stats[str(episode_num)][\"SM_1\"][\"raw_observations\"][ii][\"rgba\"])\n",
    "    patch_obs.append(detailed_stats[str(episode_num)][\"SM_0\"][\"raw_observations\"][ii][\"rgba\"])\n",
    "\n",
    "viz_obs = np.array(viz_obs) #[:100]\n",
    "patch_obs = np.array(patch_obs) #[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9817049f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def local_plot_sample_animation(patch_obs, viz_obs, object_id=\"\", resolution=64):\n",
    "    \"\"\"Plot video of sampled oservations.\"\"\"\n",
    "    from IPython import display\n",
    "    from matplotlib import animation\n",
    "\n",
    "    fig = plt.figure(figsize=(8, 4))\n",
    "    ax1 = fig.add_subplot(1, 2, 1)\n",
    "    marked_obs = viz_obs[0].copy()\n",
    "    \n",
    "    pixel_window = int(resolution * 0.05)\n",
    "    lower_end = int(resolution/2)-pixel_window\n",
    "    upper_end = int(resolution/2)+pixel_window\n",
    "\n",
    "    marked_obs[lower_end:upper_end, lower_end:upper_end] = [0, 0, 255, 255]\n",
    "    im1 = plt.imshow(marked_obs)\n",
    "    ax1.set_xticks([]), ax1.set_yticks([])\n",
    "    plt.title(\"Overview (Zoomed out)\")\n",
    "    ax2 = fig.add_subplot(1, 2, 2)\n",
    "    im2 = plt.imshow(patch_obs[0])\n",
    "    plt.title(\"Sensor View\")\n",
    "    ax2.set_xticks([]), ax2.set_yticks([])\n",
    "\n",
    "#     num_steps = len(all_obs)\n",
    "#     plot_obs = all_obs[0]\n",
    "#     for obs in all_obs[1:]:\n",
    "#         # obj_obs = obs[np.where(obs[:, 3] > 0)]\n",
    "#         plot_obs = np.append(plot_obs, obs, axis=0)\n",
    "#     res = plot_obs.shape[0] // num_steps\n",
    "#     obj_obs = plot_obs[\n",
    "#         np.where((plot_obs[:res, 3] > 0))  # & (plot_obs[:res, 2] < 0))\n",
    "#     ]\n",
    "\n",
    "#     scale_obs = plot_obs[np.where((plot_obs[:, 3] > 0))]\n",
    "#     p1 = ax3.scatter(\n",
    "#         -obj_obs[:, 1],\n",
    "#         obj_obs[:, 0],\n",
    "#         obj_obs[:, 2],\n",
    "#         c=obj_obs[:, 2],\n",
    "#         vmin=min(scale_obs[:, 2]),\n",
    "#         vmax=max(scale_obs[:, 2]),\n",
    "#     )\n",
    "\n",
    "#     ax3.set_xticks([]), ax3.set_yticks([]), ax3.set_zticks([])\n",
    "#     ax3.set_xlabel(\"x\", labelpad=-10)\n",
    "#     ax3.set_ylabel(\"y\", labelpad=-10)\n",
    "#     ax3.set_zlabel(\"z\", labelpad=-10)\n",
    "\n",
    "#     plot_zoom = 0.07\n",
    "#     means = np.mean(plot_obs, axis=0)\n",
    "#     ax3.set_xlim([-means[1] - plot_zoom, -means[1] + plot_zoom])\n",
    "#     ax3.set_ylim([means[0] - plot_zoom, means[0] + plot_zoom])\n",
    "#     ax3.set_zlim([means[2] - plot_zoom, means[2] + plot_zoom])\n",
    "#     ax3.view_init(110, 0)\n",
    "\n",
    "    def init():\n",
    "        # avoid calling 0 twice\n",
    "        pass\n",
    "\n",
    "    def animate(i):\n",
    "        marked_obs = viz_obs[i].copy()\n",
    "        marked_obs[lower_end:upper_end, lower_end:upper_end] = [0, 0, 255, 255]\n",
    "        im1.set_array(marked_obs)\n",
    "        im2.set_array(patch_obs[i])\n",
    "\n",
    "#         point_idx = int((i + 1) * res)\n",
    "#         obj_obs = plot_obs[\n",
    "#             np.where(\n",
    "#                 (plot_obs[:point_idx, 3] > 0)  # & (plot_obs[:point_idx, 2] < 0)\n",
    "#             )\n",
    "#         ]\n",
    "#         p1._offsets3d = (-obj_obs[:, 1], obj_obs[:, 0], obj_obs[:, 2])\n",
    "#         p1.set_array(obj_obs[:, 2])\n",
    "\n",
    "        return (ax1,)\n",
    "\n",
    "    anim = animation.FuncAnimation(\n",
    "        fig, animate, frames=len(viz_obs), init_func=init\n",
    "    )\n",
    "    anim.save(\n",
    "            \"viewfinder_\" + object_id + \".gif\",\n",
    "            writer=\"imagemagick\",\n",
    "            dpi=300,\n",
    "        )\n",
    "    video = anim.to_html5_video()\n",
    "    html = display.HTML(video)\n",
    "    display.display(html)\n",
    "    plt.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f307b324",
   "metadata": {},
   "outputs": [],
   "source": [
    "local_plot_sample_animation(patch_obs, viz_obs, \n",
    "                            object_id=detailed_stats[str(episode_num)][\"target\"][\"target_object\"],\n",
    "                           resolution=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e9e3904",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77c60b23",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}