antbullet_panda.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8daa82d0b3d04077a2da05e1c1b77320": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6d5f3cee6ee2477ba50e5a2fcb40f658",
              "IPY_MODEL_dafe16625f51423e93184de71cd0d3b1",
              "IPY_MODEL_ff516564269f44f2bc24b57385a38f28",
              "IPY_MODEL_6c1f4085057a4751a1d10ce2be4aba6a",
              "IPY_MODEL_7b60a431d4aa46e6a439786be053ddd5"
            ],
            "layout": "IPY_MODEL_34560688d0954fbcbfac6ef53b4a9158"
          }
        },
        "6d5f3cee6ee2477ba50e5a2fcb40f658": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a54cfeed81e74dd5bdb42224bec7f106",
            "placeholder": "​",
            "style": "IPY_MODEL_7e0d0e5d3262438eafaafa1c53745eb4",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "dafe16625f51423e93184de71cd0d3b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_0b73207b81c34e5394fa2c9dfe8787cd",
            "placeholder": "​",
            "style": "IPY_MODEL_b5fd7b531aeb43ac917403aacd9b9d06",
            "value": ""
          }
        },
        "ff516564269f44f2bc24b57385a38f28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_ea5047dd899f4ef4a0319e479dbdeaef",
            "style": "IPY_MODEL_f867edb6e9674597a97f7bdf2394b004",
            "value": true
          }
        },
        "6c1f4085057a4751a1d10ce2be4aba6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_6d7f165380b040ae934d805db9245825",
            "style": "IPY_MODEL_ec469f3d62714626ac6d525061da01c4",
            "tooltip": ""
          }
        },
        "7b60a431d4aa46e6a439786be053ddd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b05518858f14b9d955af42e4f9088b9",
            "placeholder": "​",
            "style": "IPY_MODEL_560749b5170149c39c73d33dc0c50f0f",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "34560688d0954fbcbfac6ef53b4a9158": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "a54cfeed81e74dd5bdb42224bec7f106": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e0d0e5d3262438eafaafa1c53745eb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0b73207b81c34e5394fa2c9dfe8787cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5fd7b531aeb43ac917403aacd9b9d06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ea5047dd899f4ef4a0319e479dbdeaef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f867edb6e9674597a97f7bdf2394b004": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6d7f165380b040ae934d805db9245825": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec469f3d62714626ac6d525061da01c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "4b05518858f14b9d955af42e4f9088b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "560749b5170149c39c73d33dc0c50f0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "82e17bdfe46c418f94ba6d0fd8d5db73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8f6751d08c17414f96b2194d770adf09",
              "IPY_MODEL_a004d15844f049aeb404143c9080799a",
              "IPY_MODEL_aa6969be7ead4649ad1a6da583b3f7c2"
            ],
            "layout": "IPY_MODEL_c67a09a38db24eadb429806bf8000ac1"
          }
        },
        "8f6751d08c17414f96b2194d770adf09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21051fb7bf9f4dc1ae5916132b2f1fa9",
            "placeholder": "​",
            "style": "IPY_MODEL_c26d255828ff4d479aa8723000090a10",
            "value": "policy.optimizer.pth: 100%"
          }
        },
        "a004d15844f049aeb404143c9080799a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2756dbef561440339806b0207906571d",
            "max": 44734,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ebe2811bc4564eccbccfbc8293ae510b",
            "value": 44734
          }
        },
        "aa6969be7ead4649ad1a6da583b3f7c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67df91f24db846d991b086346ae82956",
            "placeholder": "​",
            "style": "IPY_MODEL_46428c01fdbb4640941b79aed148ee7d",
            "value": " 44.7k/44.7k [00:00&lt;00:00, 53.6kB/s]"
          }
        },
        "c67a09a38db24eadb429806bf8000ac1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21051fb7bf9f4dc1ae5916132b2f1fa9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c26d255828ff4d479aa8723000090a10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2756dbef561440339806b0207906571d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ebe2811bc4564eccbccfbc8293ae510b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "67df91f24db846d991b086346ae82956": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46428c01fdbb4640941b79aed148ee7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e7622a22b2d1466e97a96f1530c76af0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8801f1dac6b04097b4e303cb84329807",
              "IPY_MODEL_cebaf557c2eb4fdb83bd76951f1c93ef",
              "IPY_MODEL_e9661091ad5447bc850bd1530705b7a7"
            ],
            "layout": "IPY_MODEL_9ff1b86161504a32871300c403c19c65"
          }
        },
        "8801f1dac6b04097b4e303cb84329807": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc926b5a4cb34df0a21b49e20f298a1c",
            "placeholder": "​",
            "style": "IPY_MODEL_c8eec08153bd40a08a8ad0b43af3ea10",
            "value": "policy.pth: 100%"
          }
        },
        "cebaf557c2eb4fdb83bd76951f1c93ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_796766469caf44f0bcc60049fcb17729",
            "max": 46014,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ebfc65884bf941758fd533ddb2dfa3fa",
            "value": 46014
          }
        },
        "e9661091ad5447bc850bd1530705b7a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_640a141d4e664ea7b3d446f01118e197",
            "placeholder": "​",
            "style": "IPY_MODEL_69aa3a08e8a44509b895861f2c11eaed",
            "value": " 46.0k/46.0k [00:00&lt;00:00, 50.6kB/s]"
          }
        },
        "9ff1b86161504a32871300c403c19c65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc926b5a4cb34df0a21b49e20f298a1c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8eec08153bd40a08a8ad0b43af3ea10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "796766469caf44f0bcc60049fcb17729": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ebfc65884bf941758fd533ddb2dfa3fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "640a141d4e664ea7b3d446f01118e197": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69aa3a08e8a44509b895861f2c11eaed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f5626fff7bf47aebdccc146eb51a53d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4241d484a1c745399b041a1f01f00e85",
              "IPY_MODEL_b7cefd736ac0410398a2101ab623ace3",
              "IPY_MODEL_4851f5860f3840a184efbf5191b7b718"
            ],
            "layout": "IPY_MODEL_475f3c596e7f417488c66b1c857ce988"
          }
        },
        "4241d484a1c745399b041a1f01f00e85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_afcab5cc1221428baeeb6f6dbb6957c2",
            "placeholder": "​",
            "style": "IPY_MODEL_0b21d71be2b24e7c928749f692193b61",
            "value": "Upload 4 LFS files: 100%"
          }
        },
        "b7cefd736ac0410398a2101ab623ace3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3db8dadc5c9a4c5e97b1357ab2d599e5",
            "max": 4,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e129a7b74fab41a28184c0d73b9f564f",
            "value": 4
          }
        },
        "4851f5860f3840a184efbf5191b7b718": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_da94a1dd564f493f8a1edcc77085ccda",
            "placeholder": "​",
            "style": "IPY_MODEL_dbbc8e5fa5e748d2906ef046e1ecfcbb",
            "value": " 4/4 [00:00&lt;00:00,  3.22it/s]"
          }
        },
        "475f3c596e7f417488c66b1c857ce988": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "afcab5cc1221428baeeb6f6dbb6957c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b21d71be2b24e7c928749f692193b61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3db8dadc5c9a4c5e97b1357ab2d599e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e129a7b74fab41a28184c0d73b9f564f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "da94a1dd564f493f8a1edcc77085ccda": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbbc8e5fa5e748d2906ef046e1ecfcbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "602e35a5e3ec41328f398f1c1ab2fbfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_26ce40a4b2404720b0c5d09df046f4d5",
              "IPY_MODEL_e3668527d6d94c2c83593e9cf76ffb10",
              "IPY_MODEL_bc441b469d4a4af5a1c687c4f620be6e"
            ],
            "layout": "IPY_MODEL_2e90b814d11d4e61aa1c94084bb4da93"
          }
        },
        "26ce40a4b2404720b0c5d09df046f4d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44d573ff033f4bbda0254d8b86b7b498",
            "placeholder": "​",
            "style": "IPY_MODEL_f7f07ad990084942b0096ada6dce88a0",
            "value": "vec_normalize.pkl: 100%"
          }
        },
        "e3668527d6d94c2c83593e9cf76ffb10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_87e8824d2e974696bda9ba37255eaf7b",
            "max": 3056,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d822c1ad64554a1282c6025d7b2755b5",
            "value": 3056
          }
        },
        "bc441b469d4a4af5a1c687c4f620be6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a09d67e563a84a86a494c41b7ad28640",
            "placeholder": "​",
            "style": "IPY_MODEL_739a9f1e952a4a8097e6176c9dc7354d",
            "value": " 3.06k/3.06k [00:00&lt;00:00, 23.3kB/s]"
          }
        },
        "2e90b814d11d4e61aa1c94084bb4da93": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44d573ff033f4bbda0254d8b86b7b498": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7f07ad990084942b0096ada6dce88a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "87e8824d2e974696bda9ba37255eaf7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d822c1ad64554a1282c6025d7b2755b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a09d67e563a84a86a494c41b7ad28640": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "739a9f1e952a4a8097e6176c9dc7354d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f69a1e606634451f9a6499988bd6dca1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_828ec510aa114d2b8c281e609aba3c9e",
              "IPY_MODEL_9d47051358ee4d908d49af3a0a438005",
              "IPY_MODEL_100899cc1e7546a9ab1d6018203ef92c"
            ],
            "layout": "IPY_MODEL_4867a4e62ad44be39c47a90838d7ea72"
          }
        },
        "828ec510aa114d2b8c281e609aba3c9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c44f06ce8c1d44e59170acc57dcbf799",
            "placeholder": "​",
            "style": "IPY_MODEL_c5d0d0af031448a481c185443642d87d",
            "value": "a2c-PandaReachDense-v2.zip: 100%"
          }
        },
        "9d47051358ee4d908d49af3a0a438005": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb1d52a93e454686a2c19d115a3f135d",
            "max": 153277,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e578ec5e4d9844d1a3fbf3aecdea0c70",
            "value": 153277
          }
        },
        "100899cc1e7546a9ab1d6018203ef92c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5a1c9dfd0f714ab795debb0701d095dc",
            "placeholder": "​",
            "style": "IPY_MODEL_e20f44e3bfc54be0be9070d4d1582f2c",
            "value": " 153k/153k [00:00&lt;00:00, 61.2kB/s]"
          }
        },
        "4867a4e62ad44be39c47a90838d7ea72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c44f06ce8c1d44e59170acc57dcbf799": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5d0d0af031448a481c185443642d87d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb1d52a93e454686a2c19d115a3f135d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e578ec5e4d9844d1a3fbf3aecdea0c70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5a1c9dfd0f714ab795debb0701d095dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e20f44e3bfc54be0be9070d4d1582f2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ryukijano/Deep-Reinforcement-Learning-and-Hugging-Face/blob/master/antbullet_panda.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Unit 6: Advantage Actor Critic (A2C) using Robotics Simulations with PyBullet and Panda-Gym 🤖\n",
        "\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/thumbnail.png\"  alt=\"Thumbnail\"/>\n",
        "\n",
        "In this notebook, you'll learn to use A2C with PyBullet and Panda-Gym, two set of robotics environments. \n",
        "\n",
        "With [PyBullet](https://github.com/bulletphysics/bullet3), you're going to **train a robot to move**:\n",
        "- `AntBulletEnv-v0` 🕸️ More precisely, a spider (they say Ant but come on... it's a spider 😆) 🕸️\n",
        "\n",
        "Then, with [Panda-Gym](https://github.com/qgallouedec/panda-gym), you're going **to train a robotic arm** (Franka Emika Panda robot) to perform a task:\n",
        "- `Reach`: the robot must place its end-effector at a target position.\n",
        "\n",
        "After that, you'll be able **to train in other robotics environments**.\n"
      ],
      "metadata": {
        "id": "-PTReiOw-RAN"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/environments.gif\" alt=\"Robotics environments\"/>"
      ],
      "metadata": {
        "id": "2VGL_0ncoAJI"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 🎮 Environments: \n",
        "\n",
        "- [PyBullet](https://github.com/bulletphysics/bullet3)\n",
        "- [Panda-Gym](https://github.com/qgallouedec/panda-gym)\n",
        "\n",
        "###📚 RL-Library: \n",
        "\n",
        "- [Stable-Baselines3](https://stable-baselines3.readthedocs.io/)"
      ],
      "metadata": {
        "id": "QInFitfWno1Q"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We're constantly trying to improve our tutorials, so **if you find some issues in this notebook**, please [open an issue on the GitHub Repo](https://github.com/huggingface/deep-rl-class/issues)."
      ],
      "metadata": {
        "id": "2CcdX4g3oFlp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Objectives of this notebook 🏆\n",
        "\n",
        "At the end of the notebook, you will:\n",
        "\n",
        "- Be able to use **PyBullet** and **Panda-Gym**, the environment libraries.\n",
        "- Be able to **train robots using A2C**.\n",
        "- Understand why **we need to normalize the input**.\n",
        "- Be able to **push your trained agent and the code to the Hub** with a nice video replay and an evaluation score 🔥.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "MoubJX20oKaQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## This notebook is from the Deep Reinforcement Learning Course\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/deep-rl-course-illustration.jpg\" alt=\"Deep RL Course illustration\"/>\n",
        "\n",
        "In this free course, you will:\n",
        "\n",
        "- 📖 Study Deep Reinforcement Learning in **theory and practice**.\n",
        "- 🧑‍💻 Learn to **use famous Deep RL libraries** such as Stable Baselines3, RL Baselines3 Zoo, CleanRL and Sample Factory 2.0.\n",
        "- 🤖 Train **agents in unique environments** \n",
        "\n",
        "And more check 📚 the syllabus 👉 https://simoninithomas.github.io/deep-rl-course\n",
        "\n",
        "Don’t forget to **<a href=\"http://eepurl.com/ic5ZUD\">sign up to the course</a>** (we are collecting your email to be able to **send you the links when each Unit is published and give you information about the challenges and updates).**\n",
        "\n",
        "\n",
        "The best way to keep in touch is to join our discord server to exchange with the community and with us 👉🏻 https://discord.gg/ydHrjt3WP5"
      ],
      "metadata": {
        "id": "DoUNkTExoUED"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Prerequisites 🏗️\n",
        "Before diving into the notebook, you need to:\n",
        "\n",
        "🔲 📚 Study [Actor-Critic methods by reading Unit 6](https://huggingface.co/deep-rl-course/unit6/introduction) 🤗  "
      ],
      "metadata": {
        "id": "BTuQAUAPoa5E"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Let's train our first robots 🤖"
      ],
      "metadata": {
        "id": "iajHvVDWoo01"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "To validate this hands-on for the [certification process](https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process),  you need to push your two trained models to the Hub and get the following results:\n",
        "\n",
        "- `AntBulletEnv-v0` get a result of >= 650.\n",
        "- `PandaReachDense-v2` get a result of >= -3.5.\n",
        "\n",
        "To find your result, go to the [leaderboard](https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard) and find your model, **the result = mean_reward - std of reward**\n",
        "\n",
        "If you don't find your model, **go to the bottom of the page and click on the refresh button**\n",
        "\n",
        "For more information about the certification process, check this section 👉 https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"
      ],
      "metadata": {
        "id": "zbOENTE2os_D"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Set the GPU 💪\n",
        "- To **accelerate the agent's training, we'll use a GPU**. To do that, go to `Runtime > Change Runtime type`\n",
        "\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg\" alt=\"GPU Step 1\">"
      ],
      "metadata": {
        "id": "PU4FVzaoM6fC"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- `Hardware Accelerator > GPU`\n",
        "\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg\" alt=\"GPU Step 2\">"
      ],
      "metadata": {
        "id": "KV0NyFdQM9ZG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create a virtual display 🔽\n",
        "\n",
        "During the notebook, we'll need to generate a replay video. To do so, with colab, **we need to have a virtual screen to be able to render the environment** (and thus record the frames). \n",
        "\n",
        "Hence the following cell will install the librairies and create and run a virtual screen 🖥"
      ],
      "metadata": {
        "id": "bTpYcVZVMzUI"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jV6wjQ7Be7p5"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!apt install python-opengl\n",
        "!apt install ffmpeg\n",
        "!apt install xvfb\n",
        "!pip3 install pyvirtualdisplay"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Virtual display\n",
        "from pyvirtualdisplay import Display\n",
        "\n",
        "virtual_display = Display(visible=0, size=(1400, 900))\n",
        "virtual_display.start()"
      ],
      "metadata": {
        "id": "ww5PQH1gNLI4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e4c8dae-3ee6-44e4-afaa-3d700785719e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyvirtualdisplay.display.Display at 0x7f8f08bb8f10>"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Install dependencies 🔽\n",
        "The first step is to install the dependencies, we’ll install multiple ones:\n",
        "\n",
        "- `pybullet`: Contains the walking robots environments.\n",
        "- `panda-gym`: Contains the robotics arm environments.\n",
        "- `stable-baselines3[extra]`: The SB3 deep reinforcement learning library.\n",
        "- `huggingface_sb3`: Additional code for Stable-baselines3 to load and upload models from the Hugging Face 🤗 Hub.\n",
        "- `huggingface_hub`: Library allowing anyone to work with the Hub repositories."
      ],
      "metadata": {
        "id": "e1obkbdJ_KnG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2yZRi_0bQGPM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ebe94ade-7f11-4559-fd08-7bc14a1f2608"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: stable-baselines3[extra] in /usr/local/lib/python3.9/dist-packages (from -r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.7.0)\n",
            "Requirement already satisfied: huggingface_sb3 in /usr/local/lib/python3.9/dist-packages (from -r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 2)) (2.2.4)\n",
            "Requirement already satisfied: panda_gym==2.0.0 in /usr/local/lib/python3.9/dist-packages (from -r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 3)) (2.0.0)\n",
            "Requirement already satisfied: pyglet==1.5.1 in /usr/local/lib/python3.9/dist-packages (from -r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 4)) (1.5.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from panda_gym==2.0.0->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 3)) (1.22.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.9/dist-packages (from panda_gym==2.0.0->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 3)) (1.10.1)\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.9/dist-packages (from panda_gym==2.0.0->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 3)) (0.21.0)\n",
            "Requirement already satisfied: pybullet in /usr/local/lib/python3.9/dist-packages (from panda_gym==2.0.0->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 3)) (3.2.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.3.5)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (3.5.3)\n",
            "Requirement already satisfied: torch>=1.11 in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.13.1+cu116)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.2.1)\n",
            "Requirement already satisfied: importlib-metadata~=4.13 in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (4.13.0)\n",
            "Requirement already satisfied: tensorboard>=2.9.1 in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.11.2)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (4.6.0.66)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (5.4.8)\n",
            "Requirement already satisfied: ale-py==0.7.4 in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.7.4)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (8.4.0)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (13.3.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (4.65.0)\n",
            "Requirement already satisfied: autorom[accept-rom-license]~=0.4.2 in /usr/local/lib/python3.9/dist-packages (from stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.4.2)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.9/dist-packages (from ale-py==0.7.4->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (5.12.0)\n",
            "Requirement already satisfied: huggingface-hub~=0.8 in /usr/local/lib/python3.9/dist-packages (from huggingface_sb3->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 2)) (0.13.1)\n",
            "Requirement already satisfied: pyyaml~=6.0 in /usr/local/lib/python3.9/dist-packages (from huggingface_sb3->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 2)) (6.0)\n",
            "Requirement already satisfied: wasabi in /usr/local/lib/python3.9/dist-packages (from huggingface_sb3->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 2)) (0.10.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.9/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (8.1.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.25.1)\n",
            "Requirement already satisfied: AutoROM.accept-rom-license in /usr/local/lib/python3.9/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.5.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub~=0.8->huggingface_sb3->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 2)) (3.9.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub~=0.8->huggingface_sb3->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 2)) (23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub~=0.8->huggingface_sb3->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 2)) (4.5.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/dist-packages (from importlib-metadata~=4.13->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (3.15.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.8.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (3.4.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.4.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.38.4)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.16.2)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.6.1)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.51.3)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (57.4.0)\n",
            "Requirement already satisfied: protobuf<4,>=3.9.2 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (3.19.6)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.2.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.9/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.4.6)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.9/dist-packages (from matplotlib->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.11.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.9/dist-packages (from matplotlib->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.8.2)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.9/dist-packages (from matplotlib->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (4.39.0)\n",
            "Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (3.0.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.4.4)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.9/dist-packages (from pandas->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2022.7.1)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.9/dist-packages (from rich->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.14.0)\n",
            "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /usr/local/lib/python3.9/dist-packages (from rich->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.2.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.9/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.2.8)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.9/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.15.0)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (5.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.9/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.9/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.3.1)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.9/dist-packages (from markdown-it-py<3.0.0,>=2.2.0->rich->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.1.2)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (4.0.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (1.26.14)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.10)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.9/dist-packages (from werkzeug>=1.0.1->tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.1.2)\n",
            "Requirement already satisfied: libtorrent in /usr/local/lib/python3.9/dist-packages (from AutoROM.accept-rom-license->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (2.0.7)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.9/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.9/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->stable-baselines3[extra]->-r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt (line 1)) (3.2.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install -r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit6/requirements-unit6.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import the packages 📦"
      ],
      "metadata": {
        "id": "QTep3PQQABLr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pybullet_envs\n",
        "import panda_gym\n",
        "import gym\n",
        "\n",
        "import os\n",
        "\n",
        "from huggingface_sb3 import load_from_hub, package_to_hub\n",
        "\n",
        "from stable_baselines3 import A2C\n",
        "from stable_baselines3.common.evaluation import evaluate_policy\n",
        "from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize\n",
        "from stable_baselines3.common.env_util import make_vec_env\n",
        "\n",
        "from huggingface_hub import notebook_login"
      ],
      "metadata": {
        "id": "HpiB8VdnQ7Bk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Environment 1: AntBulletEnv-v0 🕸\n",
        "\n"
      ],
      "metadata": {
        "id": "lfBwIS_oAVXI"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Create the AntBulletEnv-v0\n",
        "#### The environment 🎮\n",
        "In this environment, the agent needs to use correctly its different joints to walk correctly.\n",
        "You can find a detailled explanation of this environment here: https://hackmd.io/@jeffreymo/SJJrSJh5_#PyBullet"
      ],
      "metadata": {
        "id": "frVXOrnlBerQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "env_id = \"AntBulletEnv-v0\"\n",
        "# Create the env\n",
        "env = gym.make(env_id)\n",
        "\n",
        "# Get the state space and action space\n",
        "s_size = env.observation_space.shape[0]\n",
        "a_size = env.action_space"
      ],
      "metadata": {
        "id": "JpU-JCDQYYax"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"_____OBSERVATION SPACE_____ \\n\")\n",
        "print(\"The State Space is: \", s_size)\n",
        "print(\"Sample observation\", env.observation_space.sample()) # Get a random observation"
      ],
      "metadata": {
        "id": "2ZfvcCqEYgrg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bb55a26e-dc38-48d5-ece6-00ce5e4ac21e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "_____OBSERVATION SPACE_____ \n",
            "\n",
            "The State Space is:  28\n",
            "Sample observation [ 0.9079969   0.44507018  0.18524839 -0.29020578  0.9130252  -0.5412519\n",
            " -0.00579384 -0.8938713  -0.85982805  0.38960713  0.17623445  0.7413489\n",
            " -1.3120481   0.03487071  0.9744689  -0.12380921 -0.54346013  1.9468037\n",
            " -1.2590312   1.4217961  -0.3373394   1.7405407  -1.4088     -0.7180425\n",
            " -2.1430063   0.07882362 -0.8116915   0.08001423]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The observation Space (from [Jeffrey Y Mo](https://hackmd.io/@jeffreymo/SJJrSJh5_#PyBullet)):\n",
        "\n",
        "The difference is that our observation space is 28 not 29.\n",
        "\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/obs_space.png\" alt=\"PyBullet Ant Obs space\"/>\n"
      ],
      "metadata": {
        "id": "QzMmsdMJS7jh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"\\n _____ACTION SPACE_____ \\n\")\n",
        "print(\"The Action Space is: \", a_size)\n",
        "print(\"Action Space Sample\", env.action_space.sample()) # Take a random action"
      ],
      "metadata": {
        "id": "Tc89eLTYYkK2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cf6de60f-9038-4def-c934-2e67d793da45"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " _____ACTION SPACE_____ \n",
            "\n",
            "The Action Space is:  Box([-1. -1. -1. -1. -1. -1. -1. -1.], [1. 1. 1. 1. 1. 1. 1. 1.], (8,), float32)\n",
            "Action Space Sample [ 0.7409361   0.6281361   0.7241572   0.6204377  -0.7129181   0.86028594\n",
            "  0.25823456  0.71500236]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The action Space (from [Jeffrey Y Mo](https://hackmd.io/@jeffreymo/SJJrSJh5_#PyBullet)):\n",
        "\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/action_space.png\" alt=\"PyBullet Ant Obs space\"/>\n"
      ],
      "metadata": {
        "id": "3RfsHhzZS9Pw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Normalize observation and rewards"
      ],
      "metadata": {
        "id": "S5sXcg469ysB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "A good practice in reinforcement learning is to [normalize input features](https://stable-baselines3.readthedocs.io/en/master/guide/rl_tips.html). \n",
        "\n",
        "For that purpose, there is a wrapper that will compute a running average and standard deviation of input features.\n",
        "\n",
        "We also normalize rewards with this same wrapper by adding `norm_reward = True`\n",
        "\n",
        "[You should check the documentation to fill this cell](https://stable-baselines3.readthedocs.io/en/master/guide/vec_envs.html#vecnormalize)"
      ],
      "metadata": {
        "id": "1ZyX6qf3Zva9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "env = make_vec_env(env_id, n_envs=4)\n",
        "\n",
        "# Adding this wrapper to normalize the observation and the reward\n",
        "env = VecNormalize(env, norm_obs = True, norm_reward = True, clip_obs=10.)"
      ],
      "metadata": {
        "id": "1RsDtHHAQ9Ie"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Solution"
      ],
      "metadata": {
        "id": "tF42HvI7-gs5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "env = make_vec_env(env_id, n_envs=4)\n",
        "\n",
        "env = VecNormalize(env, norm_obs=True, norm_reward=True, clip_obs=10.)"
      ],
      "metadata": {
        "id": "2O67mqgC-hol"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Create the A2C Model 🤖\n",
        "\n",
        "In this case, because we have a vector of 28 values as input, we'll use an MLP (multi-layer perceptron) as policy.\n",
        "\n",
        "For more information about A2C implementation with StableBaselines3 check: https://stable-baselines3.readthedocs.io/en/master/modules/a2c.html#notes\n",
        "\n",
        "To find the best parameters I checked the [official trained agents by Stable-Baselines3 team](https://huggingface.co/sb3)."
      ],
      "metadata": {
        "id": "4JmEVU6z1ZA-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = A2C(policy = \"MlpPolicy\",\n",
        "            env = env,\n",
        "            gae_lambda = 0.9,\n",
        "            gamma = 0.99,\n",
        "            learning_rate = 0.00096,\n",
        "            max_grad_norm = 0.5,\n",
        "            n_steps = 8,\n",
        "            vf_coef = 0.4,\n",
        "            ent_coef = 0.0,\n",
        "            policy_kwargs=dict(\n",
        "            log_std_init=-2, ortho_init=False),\n",
        "            normalize_advantage=False,\n",
        "            use_rms_prop= True,\n",
        "            use_sde= True,\n",
        "            verbose=1)"
      ],
      "metadata": {
        "id": "vR3T4qFt164I",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bc5d2baf-01f1-4223-9845-4467c1d6751c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cuda device\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Solution"
      ],
      "metadata": {
        "id": "nWAuOOLh-oQf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = A2C(policy = \"MlpPolicy\",\n",
        "            env = env,\n",
        "            gae_lambda = 0.9,\n",
        "            gamma = 0.99,\n",
        "            learning_rate = 0.00096,\n",
        "            max_grad_norm = 0.5,\n",
        "            n_steps = 8,\n",
        "            vf_coef = 0.4,\n",
        "            ent_coef = 0.0,\n",
        "            policy_kwargs=dict(\n",
        "            log_std_init=-2, ortho_init=False),\n",
        "            normalize_advantage=False,\n",
        "            use_rms_prop= True,\n",
        "            use_sde= True,\n",
        "            verbose=1)"
      ],
      "metadata": {
        "id": "FKFLY54T-pU1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Train the A2C agent 🏃\n",
        "- Let's train our agent for 2,000,000 timesteps, don't forget to use GPU on Colab. It will take approximately ~25-40min"
      ],
      "metadata": {
        "id": "opyK3mpJ1-m9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.learn(2_000_000)"
      ],
      "metadata": {
        "id": "4TuGHZD7RF1G",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6c67aefa-59a9-4540-9500-f17ca38540c4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1;30;43mStreaming output truncated to the last 5000 lines.\u001b[0m\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 34800    |\n",
            "|    time_elapsed       | 2308     |\n",
            "|    total_timesteps    | 1113600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.05    |\n",
            "|    explained_variance | 0.983    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 34799    |\n",
            "|    policy_loss        | 0.131    |\n",
            "|    std                | 0.0688   |\n",
            "|    value_loss         | 0.0074   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.44e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 34900    |\n",
            "|    time_elapsed       | 2314     |\n",
            "|    total_timesteps    | 1116800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.16    |\n",
            "|    explained_variance | 0.977    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 34899    |\n",
            "|    policy_loss        | -0.261   |\n",
            "|    std                | 0.0686   |\n",
            "|    value_loss         | 0.0114   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.45e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35000    |\n",
            "|    time_elapsed       | 2321     |\n",
            "|    total_timesteps    | 1120000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.48    |\n",
            "|    explained_variance | 0.97     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 34999    |\n",
            "|    policy_loss        | -0.0735  |\n",
            "|    std                | 0.0685   |\n",
            "|    value_loss         | 0.00525  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.45e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35100    |\n",
            "|    time_elapsed       | 2327     |\n",
            "|    total_timesteps    | 1123200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.24    |\n",
            "|    explained_variance | 0.98     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35099    |\n",
            "|    policy_loss        | 0.14     |\n",
            "|    std                | 0.0681   |\n",
            "|    value_loss         | 0.00713  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.48e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35200    |\n",
            "|    time_elapsed       | 2334     |\n",
            "|    total_timesteps    | 1126400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.97    |\n",
            "|    explained_variance | 0.9      |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35199    |\n",
            "|    policy_loss        | -0.104   |\n",
            "|    std                | 0.0681   |\n",
            "|    value_loss         | 0.00142  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.5e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35300    |\n",
            "|    time_elapsed       | 2340     |\n",
            "|    total_timesteps    | 1129600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.1     |\n",
            "|    explained_variance | 1        |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35299    |\n",
            "|    policy_loss        | 0.124    |\n",
            "|    std                | 0.0683   |\n",
            "|    value_loss         | 0.0012   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.48e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35400    |\n",
            "|    time_elapsed       | 2346     |\n",
            "|    total_timesteps    | 1132800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.8     |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35399    |\n",
            "|    policy_loss        | -0.0441  |\n",
            "|    std                | 0.0679   |\n",
            "|    value_loss         | 0.0071   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.49e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35500    |\n",
            "|    time_elapsed       | 2353     |\n",
            "|    total_timesteps    | 1136000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.03    |\n",
            "|    explained_variance | 0.959    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35499    |\n",
            "|    policy_loss        | 0.046    |\n",
            "|    std                | 0.0678   |\n",
            "|    value_loss         | 0.00106  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.49e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35600    |\n",
            "|    time_elapsed       | 2359     |\n",
            "|    total_timesteps    | 1139200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.06    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35599    |\n",
            "|    policy_loss        | 0.254    |\n",
            "|    std                | 0.068    |\n",
            "|    value_loss         | 0.00628  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.5e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35700    |\n",
            "|    time_elapsed       | 2366     |\n",
            "|    total_timesteps    | 1142400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.53    |\n",
            "|    explained_variance | 0.99     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35699    |\n",
            "|    policy_loss        | 0.0458   |\n",
            "|    std                | 0.068    |\n",
            "|    value_loss         | 0.0087   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.48e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35800    |\n",
            "|    time_elapsed       | 2374     |\n",
            "|    total_timesteps    | 1145600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.52    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35799    |\n",
            "|    policy_loss        | 0.134    |\n",
            "|    std                | 0.0679   |\n",
            "|    value_loss         | 0.00468  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.46e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 35900    |\n",
            "|    time_elapsed       | 2381     |\n",
            "|    total_timesteps    | 1148800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.84    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35899    |\n",
            "|    policy_loss        | 0.0844   |\n",
            "|    std                | 0.0677   |\n",
            "|    value_loss         | 0.00173  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.46e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36000    |\n",
            "|    time_elapsed       | 2387     |\n",
            "|    total_timesteps    | 1152000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.41    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 35999    |\n",
            "|    policy_loss        | 0.0902   |\n",
            "|    std                | 0.0673   |\n",
            "|    value_loss         | 0.00175  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.45e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36100    |\n",
            "|    time_elapsed       | 2393     |\n",
            "|    total_timesteps    | 1155200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.85    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36099    |\n",
            "|    policy_loss        | -0.144   |\n",
            "|    std                | 0.0674   |\n",
            "|    value_loss         | 0.00622  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.41e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36200    |\n",
            "|    time_elapsed       | 2400     |\n",
            "|    total_timesteps    | 1158400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.9     |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36199    |\n",
            "|    policy_loss        | 0.131    |\n",
            "|    std                | 0.0672   |\n",
            "|    value_loss         | 0.00524  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.39e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36300    |\n",
            "|    time_elapsed       | 2406     |\n",
            "|    total_timesteps    | 1161600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.01    |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36299    |\n",
            "|    policy_loss        | 0.0627   |\n",
            "|    std                | 0.0673   |\n",
            "|    value_loss         | 0.0303   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36400    |\n",
            "|    time_elapsed       | 2413     |\n",
            "|    total_timesteps    | 1164800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.71    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36399    |\n",
            "|    policy_loss        | 0.0324   |\n",
            "|    std                | 0.0669   |\n",
            "|    value_loss         | 0.00335  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36500    |\n",
            "|    time_elapsed       | 2419     |\n",
            "|    total_timesteps    | 1168000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.84    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36499    |\n",
            "|    policy_loss        | -0.103   |\n",
            "|    std                | 0.0671   |\n",
            "|    value_loss         | 0.00603  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.34e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36600    |\n",
            "|    time_elapsed       | 2426     |\n",
            "|    total_timesteps    | 1171200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.99    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36599    |\n",
            "|    policy_loss        | 0.124    |\n",
            "|    std                | 0.0673   |\n",
            "|    value_loss         | 0.00702  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.34e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36700    |\n",
            "|    time_elapsed       | 2432     |\n",
            "|    total_timesteps    | 1174400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.95    |\n",
            "|    explained_variance | 0.92     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36699    |\n",
            "|    policy_loss        | -0.453   |\n",
            "|    std                | 0.0672   |\n",
            "|    value_loss         | 0.0263   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36800    |\n",
            "|    time_elapsed       | 2438     |\n",
            "|    total_timesteps    | 1177600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.83    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36799    |\n",
            "|    policy_loss        | -0.0742  |\n",
            "|    std                | 0.0669   |\n",
            "|    value_loss         | 0.00237  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 36900    |\n",
            "|    time_elapsed       | 2445     |\n",
            "|    total_timesteps    | 1180800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.88    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36899    |\n",
            "|    policy_loss        | -0.0238  |\n",
            "|    std                | 0.0668   |\n",
            "|    value_loss         | 0.000575 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.34e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37000    |\n",
            "|    time_elapsed       | 2451     |\n",
            "|    total_timesteps    | 1184000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.88    |\n",
            "|    explained_variance | 0.614    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 36999    |\n",
            "|    policy_loss        | 0.0788   |\n",
            "|    std                | 0.0669   |\n",
            "|    value_loss         | 0.00189  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.34e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 482      |\n",
            "|    iterations         | 37100    |\n",
            "|    time_elapsed       | 2458     |\n",
            "|    total_timesteps    | 1187200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5       |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37099    |\n",
            "|    policy_loss        | 0.0101   |\n",
            "|    std                | 0.067    |\n",
            "|    value_loss         | 0.00279  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.33e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37200    |\n",
            "|    time_elapsed       | 2464     |\n",
            "|    total_timesteps    | 1190400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.19    |\n",
            "|    explained_variance | 0.575    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37199    |\n",
            "|    policy_loss        | 0.0808   |\n",
            "|    std                | 0.0674   |\n",
            "|    value_loss         | 0.000333 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37300    |\n",
            "|    time_elapsed       | 2470     |\n",
            "|    total_timesteps    | 1193600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.14    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37299    |\n",
            "|    policy_loss        | 0.00256  |\n",
            "|    std                | 0.0671   |\n",
            "|    value_loss         | 0.00186  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37400    |\n",
            "|    time_elapsed       | 2477     |\n",
            "|    total_timesteps    | 1196800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.77    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37399    |\n",
            "|    policy_loss        | -0.00222 |\n",
            "|    std                | 0.0671   |\n",
            "|    value_loss         | 0.000298 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37500    |\n",
            "|    time_elapsed       | 2482     |\n",
            "|    total_timesteps    | 1200000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.93    |\n",
            "|    explained_variance | 0.545    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37499    |\n",
            "|    policy_loss        | 0.0815   |\n",
            "|    std                | 0.0669   |\n",
            "|    value_loss         | 0.00447  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.36e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37600    |\n",
            "|    time_elapsed       | 2490     |\n",
            "|    total_timesteps    | 1203200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.81    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37599    |\n",
            "|    policy_loss        | 0.0529   |\n",
            "|    std                | 0.0667   |\n",
            "|    value_loss         | 0.00206  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37700    |\n",
            "|    time_elapsed       | 2496     |\n",
            "|    total_timesteps    | 1206400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.85    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37699    |\n",
            "|    policy_loss        | 0.00415  |\n",
            "|    std                | 0.0663   |\n",
            "|    value_loss         | 0.00143  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37800    |\n",
            "|    time_elapsed       | 2503     |\n",
            "|    total_timesteps    | 1209600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.75    |\n",
            "|    explained_variance | 0.876    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37799    |\n",
            "|    policy_loss        | 0.205    |\n",
            "|    std                | 0.0665   |\n",
            "|    value_loss         | 0.00603  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 37900    |\n",
            "|    time_elapsed       | 2509     |\n",
            "|    total_timesteps    | 1212800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.75    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37899    |\n",
            "|    policy_loss        | 0.131    |\n",
            "|    std                | 0.0662   |\n",
            "|    value_loss         | 0.00141  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.36e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38000    |\n",
            "|    time_elapsed       | 2515     |\n",
            "|    total_timesteps    | 1216000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.06    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 37999    |\n",
            "|    policy_loss        | 0.0422   |\n",
            "|    std                | 0.066    |\n",
            "|    value_loss         | 0.00111  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.36e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38100    |\n",
            "|    time_elapsed       | 2522     |\n",
            "|    total_timesteps    | 1219200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.43    |\n",
            "|    explained_variance | 0.98     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38099    |\n",
            "|    policy_loss        | 0.00407  |\n",
            "|    std                | 0.0661   |\n",
            "|    value_loss         | 0.000983 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38200    |\n",
            "|    time_elapsed       | 2527     |\n",
            "|    total_timesteps    | 1222400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.93    |\n",
            "|    explained_variance | 0.945    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38199    |\n",
            "|    policy_loss        | -0.0625  |\n",
            "|    std                | 0.0664   |\n",
            "|    value_loss         | 0.000898 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38300    |\n",
            "|    time_elapsed       | 2535     |\n",
            "|    total_timesteps    | 1225600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.15    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38299    |\n",
            "|    policy_loss        | -0.0745  |\n",
            "|    std                | 0.0663   |\n",
            "|    value_loss         | 0.00303  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 992      |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38400    |\n",
            "|    time_elapsed       | 2540     |\n",
            "|    total_timesteps    | 1228800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.99    |\n",
            "|    explained_variance | 0.544    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38399    |\n",
            "|    policy_loss        | -0.57    |\n",
            "|    std                | 0.0667   |\n",
            "|    value_loss         | 0.105    |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.4e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38500    |\n",
            "|    time_elapsed       | 2547     |\n",
            "|    total_timesteps    | 1232000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.7     |\n",
            "|    explained_variance | 0.977    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38499    |\n",
            "|    policy_loss        | -0.0126  |\n",
            "|    std                | 0.0667   |\n",
            "|    value_loss         | 0.000525 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.41e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38600    |\n",
            "|    time_elapsed       | 2553     |\n",
            "|    total_timesteps    | 1235200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.83    |\n",
            "|    explained_variance | 0.986    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38599    |\n",
            "|    policy_loss        | 0.126    |\n",
            "|    std                | 0.0667   |\n",
            "|    value_loss         | 0.00253  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.42e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38700    |\n",
            "|    time_elapsed       | 2559     |\n",
            "|    total_timesteps    | 1238400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.99    |\n",
            "|    explained_variance | 0.98     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38699    |\n",
            "|    policy_loss        | 0.169    |\n",
            "|    std                | 0.0665   |\n",
            "|    value_loss         | 0.00546  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.42e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38800    |\n",
            "|    time_elapsed       | 2566     |\n",
            "|    total_timesteps    | 1241600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.11    |\n",
            "|    explained_variance | 0.813    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38799    |\n",
            "|    policy_loss        | -0.0191  |\n",
            "|    std                | 0.0664   |\n",
            "|    value_loss         | 0.000921 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.45e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 38900    |\n",
            "|    time_elapsed       | 2572     |\n",
            "|    total_timesteps    | 1244800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.71    |\n",
            "|    explained_variance | 0.99     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38899    |\n",
            "|    policy_loss        | -0.0997  |\n",
            "|    std                | 0.066    |\n",
            "|    value_loss         | 0.00285  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.46e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39000    |\n",
            "|    time_elapsed       | 2580     |\n",
            "|    total_timesteps    | 1248000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.87    |\n",
            "|    explained_variance | 0.982    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 38999    |\n",
            "|    policy_loss        | -0.0736  |\n",
            "|    std                | 0.0663   |\n",
            "|    value_loss         | 0.000517 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.48e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39100    |\n",
            "|    time_elapsed       | 2587     |\n",
            "|    total_timesteps    | 1251200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.08    |\n",
            "|    explained_variance | 0.908    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39099    |\n",
            "|    policy_loss        | 0.0162   |\n",
            "|    std                | 0.0661   |\n",
            "|    value_loss         | 0.00205  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.5e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39200    |\n",
            "|    time_elapsed       | 2594     |\n",
            "|    total_timesteps    | 1254400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -5.02    |\n",
            "|    explained_variance | 0.984    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39199    |\n",
            "|    policy_loss        | 0.00773  |\n",
            "|    std                | 0.0659   |\n",
            "|    value_loss         | 0.00112  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.53e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39300    |\n",
            "|    time_elapsed       | 2600     |\n",
            "|    total_timesteps    | 1257600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.55    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39299    |\n",
            "|    policy_loss        | 0.0219   |\n",
            "|    std                | 0.066    |\n",
            "|    value_loss         | 0.00102  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.57e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39400    |\n",
            "|    time_elapsed       | 2606     |\n",
            "|    total_timesteps    | 1260800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.81    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39399    |\n",
            "|    policy_loss        | 0.139    |\n",
            "|    std                | 0.0659   |\n",
            "|    value_loss         | 0.00226  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.59e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39500    |\n",
            "|    time_elapsed       | 2613     |\n",
            "|    total_timesteps    | 1264000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.99    |\n",
            "|    explained_variance | 0.952    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39499    |\n",
            "|    policy_loss        | 0.129    |\n",
            "|    std                | 0.0659   |\n",
            "|    value_loss         | 0.00223  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.6e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39600    |\n",
            "|    time_elapsed       | 2619     |\n",
            "|    total_timesteps    | 1267200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.91    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39599    |\n",
            "|    policy_loss        | 0.131    |\n",
            "|    std                | 0.0659   |\n",
            "|    value_loss         | 0.00107  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.64e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39700    |\n",
            "|    time_elapsed       | 2626     |\n",
            "|    total_timesteps    | 1270400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.75    |\n",
            "|    explained_variance | 0.531    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39699    |\n",
            "|    policy_loss        | -0.0197  |\n",
            "|    std                | 0.0658   |\n",
            "|    value_loss         | 0.000686 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39800    |\n",
            "|    time_elapsed       | 2632     |\n",
            "|    total_timesteps    | 1273600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.88    |\n",
            "|    explained_variance | 0.782    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39799    |\n",
            "|    policy_loss        | 0.00047  |\n",
            "|    std                | 0.0655   |\n",
            "|    value_loss         | 0.00058  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 39900    |\n",
            "|    time_elapsed       | 2638     |\n",
            "|    total_timesteps    | 1276800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.97    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39899    |\n",
            "|    policy_loss        | 0.0629   |\n",
            "|    std                | 0.0652   |\n",
            "|    value_loss         | 0.00122  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.69e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 40000    |\n",
            "|    time_elapsed       | 2645     |\n",
            "|    total_timesteps    | 1280000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.84    |\n",
            "|    explained_variance | 0.989    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 39999    |\n",
            "|    policy_loss        | -0.16    |\n",
            "|    std                | 0.065    |\n",
            "|    value_loss         | 0.00138  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.69e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 40100    |\n",
            "|    time_elapsed       | 2650     |\n",
            "|    total_timesteps    | 1283200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.76    |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40099    |\n",
            "|    policy_loss        | 0.0481   |\n",
            "|    std                | 0.0651   |\n",
            "|    value_loss         | 0.00202  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.71e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 483      |\n",
            "|    iterations         | 40200    |\n",
            "|    time_elapsed       | 2658     |\n",
            "|    total_timesteps    | 1286400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.92    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40199    |\n",
            "|    policy_loss        | 0.0499   |\n",
            "|    std                | 0.0651   |\n",
            "|    value_loss         | 0.000306 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.72e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 40300    |\n",
            "|    time_elapsed       | 2663     |\n",
            "|    total_timesteps    | 1289600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.77    |\n",
            "|    explained_variance | 0.964    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40299    |\n",
            "|    policy_loss        | 0.102    |\n",
            "|    std                | 0.0652   |\n",
            "|    value_loss         | 0.00277  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 40400    |\n",
            "|    time_elapsed       | 2671     |\n",
            "|    total_timesteps    | 1292800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.66    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40399    |\n",
            "|    policy_loss        | 0.0246   |\n",
            "|    std                | 0.065    |\n",
            "|    value_loss         | 0.00106  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 40500    |\n",
            "|    time_elapsed       | 2676     |\n",
            "|    total_timesteps    | 1296000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.77    |\n",
            "|    explained_variance | 0.969    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40499    |\n",
            "|    policy_loss        | 0.21     |\n",
            "|    std                | 0.0655   |\n",
            "|    value_loss         | 0.00425  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.74e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 40600    |\n",
            "|    time_elapsed       | 2683     |\n",
            "|    total_timesteps    | 1299200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.63    |\n",
            "|    explained_variance | 0.953    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40599    |\n",
            "|    policy_loss        | -0.148   |\n",
            "|    std                | 0.0654   |\n",
            "|    value_loss         | 0.00593  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 40700    |\n",
            "|    time_elapsed       | 2689     |\n",
            "|    total_timesteps    | 1302400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.85    |\n",
            "|    explained_variance | 0.601    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40699    |\n",
            "|    policy_loss        | -0.287   |\n",
            "|    std                | 0.0653   |\n",
            "|    value_loss         | 0.0177   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.75e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 40800    |\n",
            "|    time_elapsed       | 2695     |\n",
            "|    total_timesteps    | 1305600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.85    |\n",
            "|    explained_variance | 0.988    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40799    |\n",
            "|    policy_loss        | 0.097    |\n",
            "|    std                | 0.0652   |\n",
            "|    value_loss         | 0.0023   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.75e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 40900    |\n",
            "|    time_elapsed       | 2702     |\n",
            "|    total_timesteps    | 1308800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.9     |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40899    |\n",
            "|    policy_loss        | 0.0589   |\n",
            "|    std                | 0.0652   |\n",
            "|    value_loss         | 0.00115  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41000    |\n",
            "|    time_elapsed       | 2708     |\n",
            "|    total_timesteps    | 1312000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.98    |\n",
            "|    explained_variance | 0.989    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 40999    |\n",
            "|    policy_loss        | -0.0674  |\n",
            "|    std                | 0.0651   |\n",
            "|    value_loss         | 0.000824 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41100    |\n",
            "|    time_elapsed       | 2715     |\n",
            "|    total_timesteps    | 1315200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.67    |\n",
            "|    explained_variance | 0.908    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41099    |\n",
            "|    policy_loss        | 0.0907   |\n",
            "|    std                | 0.0656   |\n",
            "|    value_loss         | 0.0132   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.77e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41200    |\n",
            "|    time_elapsed       | 2721     |\n",
            "|    total_timesteps    | 1318400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.75    |\n",
            "|    explained_variance | 0.771    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41199    |\n",
            "|    policy_loss        | -0.0864  |\n",
            "|    std                | 0.0654   |\n",
            "|    value_loss         | 0.00135  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.77e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41300    |\n",
            "|    time_elapsed       | 2727     |\n",
            "|    total_timesteps    | 1321600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.78    |\n",
            "|    explained_variance | 0.991    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41299    |\n",
            "|    policy_loss        | -0.219   |\n",
            "|    std                | 0.0655   |\n",
            "|    value_loss         | 0.0108   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41400    |\n",
            "|    time_elapsed       | 2734     |\n",
            "|    total_timesteps    | 1324800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.95    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41399    |\n",
            "|    policy_loss        | -0.0449  |\n",
            "|    std                | 0.0652   |\n",
            "|    value_loss         | 0.000885 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41500    |\n",
            "|    time_elapsed       | 2740     |\n",
            "|    total_timesteps    | 1328000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.82    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41499    |\n",
            "|    policy_loss        | 0.0644   |\n",
            "|    std                | 0.0652   |\n",
            "|    value_loss         | 0.00489  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41600    |\n",
            "|    time_elapsed       | 2747     |\n",
            "|    total_timesteps    | 1331200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.7     |\n",
            "|    explained_variance | 0.951    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41599    |\n",
            "|    policy_loss        | 0.0733   |\n",
            "|    std                | 0.0649   |\n",
            "|    value_loss         | 0.00663  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41700    |\n",
            "|    time_elapsed       | 2753     |\n",
            "|    total_timesteps    | 1334400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.76    |\n",
            "|    explained_variance | 0.729    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41699    |\n",
            "|    policy_loss        | -0.0646  |\n",
            "|    std                | 0.0646   |\n",
            "|    value_loss         | 0.000802 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41800    |\n",
            "|    time_elapsed       | 2759     |\n",
            "|    total_timesteps    | 1337600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.48    |\n",
            "|    explained_variance | 0.975    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41799    |\n",
            "|    policy_loss        | -0.0116  |\n",
            "|    std                | 0.0643   |\n",
            "|    value_loss         | 0.00221  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 41900    |\n",
            "|    time_elapsed       | 2766     |\n",
            "|    total_timesteps    | 1340800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.85    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41899    |\n",
            "|    policy_loss        | 0.0999   |\n",
            "|    std                | 0.0642   |\n",
            "|    value_loss         | 0.000937 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42000    |\n",
            "|    time_elapsed       | 2771     |\n",
            "|    total_timesteps    | 1344000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.64    |\n",
            "|    explained_variance | 0.961    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 41999    |\n",
            "|    policy_loss        | 0.0205   |\n",
            "|    std                | 0.064    |\n",
            "|    value_loss         | 0.00107  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42100    |\n",
            "|    time_elapsed       | 2778     |\n",
            "|    total_timesteps    | 1347200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.76    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42099    |\n",
            "|    policy_loss        | -0.00438 |\n",
            "|    std                | 0.0639   |\n",
            "|    value_loss         | 0.00104  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42200    |\n",
            "|    time_elapsed       | 2784     |\n",
            "|    total_timesteps    | 1350400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.89    |\n",
            "|    explained_variance | 0.976    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42199    |\n",
            "|    policy_loss        | 0.0972   |\n",
            "|    std                | 0.0641   |\n",
            "|    value_loss         | 0.00213  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42300    |\n",
            "|    time_elapsed       | 2791     |\n",
            "|    total_timesteps    | 1353600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.77    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42299    |\n",
            "|    policy_loss        | -0.165   |\n",
            "|    std                | 0.0639   |\n",
            "|    value_loss         | 0.00508  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.74e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42400    |\n",
            "|    time_elapsed       | 2798     |\n",
            "|    total_timesteps    | 1356800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.64    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42399    |\n",
            "|    policy_loss        | -0.0723  |\n",
            "|    std                | 0.0639   |\n",
            "|    value_loss         | 0.00274  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42500    |\n",
            "|    time_elapsed       | 2805     |\n",
            "|    total_timesteps    | 1360000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.21    |\n",
            "|    explained_variance | 0.989    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42499    |\n",
            "|    policy_loss        | -0.125   |\n",
            "|    std                | 0.0637   |\n",
            "|    value_loss         | 0.00926  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.72e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42600    |\n",
            "|    time_elapsed       | 2812     |\n",
            "|    total_timesteps    | 1363200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.78    |\n",
            "|    explained_variance | 0.699    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42599    |\n",
            "|    policy_loss        | -0.0652  |\n",
            "|    std                | 0.0638   |\n",
            "|    value_loss         | 0.00982  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.7e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42700    |\n",
            "|    time_elapsed       | 2818     |\n",
            "|    total_timesteps    | 1366400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.61    |\n",
            "|    explained_variance | 0.848    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42699    |\n",
            "|    policy_loss        | -0.00936 |\n",
            "|    std                | 0.0637   |\n",
            "|    value_loss         | 0.00331  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.7e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 42800    |\n",
            "|    time_elapsed       | 2825     |\n",
            "|    total_timesteps    | 1369600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.3     |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42799    |\n",
            "|    policy_loss        | 0.0245   |\n",
            "|    std                | 0.0636   |\n",
            "|    value_loss         | 0.00105  |\n",
            "------------------------------------\n",
            "-------------------------------------\n",
            "| rollout/              |           |\n",
            "|    ep_len_mean        | 1e+03     |\n",
            "|    ep_rew_mean        | 1.68e+03  |\n",
            "| time/                 |           |\n",
            "|    fps                | 484       |\n",
            "|    iterations         | 42900     |\n",
            "|    time_elapsed       | 2831      |\n",
            "|    total_timesteps    | 1372800   |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -4.42     |\n",
            "|    explained_variance | 0.985     |\n",
            "|    learning_rate      | 0.00096   |\n",
            "|    n_updates          | 42899     |\n",
            "|    policy_loss        | -0.000183 |\n",
            "|    std                | 0.0634    |\n",
            "|    value_loss         | 0.00256   |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 43000    |\n",
            "|    time_elapsed       | 2838     |\n",
            "|    total_timesteps    | 1376000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.55    |\n",
            "|    explained_variance | 0.934    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 42999    |\n",
            "|    policy_loss        | -0.0156  |\n",
            "|    std                | 0.0631   |\n",
            "|    value_loss         | 0.00139  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 43100    |\n",
            "|    time_elapsed       | 2844     |\n",
            "|    total_timesteps    | 1379200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.74    |\n",
            "|    explained_variance | 0.988    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43099    |\n",
            "|    policy_loss        | -0.045   |\n",
            "|    std                | 0.0629   |\n",
            "|    value_loss         | 0.00216  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 43200    |\n",
            "|    time_elapsed       | 2850     |\n",
            "|    total_timesteps    | 1382400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.53    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43199    |\n",
            "|    policy_loss        | -0.0431  |\n",
            "|    std                | 0.0623   |\n",
            "|    value_loss         | 0.00179  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 484      |\n",
            "|    iterations         | 43300    |\n",
            "|    time_elapsed       | 2857     |\n",
            "|    total_timesteps    | 1385600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.62    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43299    |\n",
            "|    policy_loss        | -0.0217  |\n",
            "|    std                | 0.0623   |\n",
            "|    value_loss         | 0.000583 |\n",
            "------------------------------------\n",
            "-------------------------------------\n",
            "| rollout/              |           |\n",
            "|    ep_len_mean        | 1e+03     |\n",
            "|    ep_rew_mean        | 1.67e+03  |\n",
            "| time/                 |           |\n",
            "|    fps                | 485       |\n",
            "|    iterations         | 43400     |\n",
            "|    time_elapsed       | 2862      |\n",
            "|    total_timesteps    | 1388800   |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -4.57     |\n",
            "|    explained_variance | 0.993     |\n",
            "|    learning_rate      | 0.00096   |\n",
            "|    n_updates          | 43399     |\n",
            "|    policy_loss        | -4.42e-05 |\n",
            "|    std                | 0.0625    |\n",
            "|    value_loss         | 0.00175   |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 43500    |\n",
            "|    time_elapsed       | 2869     |\n",
            "|    total_timesteps    | 1392000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.53    |\n",
            "|    explained_variance | 0.981    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43499    |\n",
            "|    policy_loss        | -0.0388  |\n",
            "|    std                | 0.0626   |\n",
            "|    value_loss         | 0.00328  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.68e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 43600    |\n",
            "|    time_elapsed       | 2875     |\n",
            "|    total_timesteps    | 1395200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.59    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43599    |\n",
            "|    policy_loss        | -0.0147  |\n",
            "|    std                | 0.0624   |\n",
            "|    value_loss         | 0.00335  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 43700    |\n",
            "|    time_elapsed       | 2882     |\n",
            "|    total_timesteps    | 1398400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.39    |\n",
            "|    explained_variance | 0.925    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43699    |\n",
            "|    policy_loss        | 0.19     |\n",
            "|    std                | 0.0623   |\n",
            "|    value_loss         | 0.00367  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 43800    |\n",
            "|    time_elapsed       | 2888     |\n",
            "|    total_timesteps    | 1401600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.41    |\n",
            "|    explained_variance | 0.922    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43799    |\n",
            "|    policy_loss        | -0.062   |\n",
            "|    std                | 0.0625   |\n",
            "|    value_loss         | 0.00158  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 43900    |\n",
            "|    time_elapsed       | 2894     |\n",
            "|    total_timesteps    | 1404800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.71    |\n",
            "|    explained_variance | 0.991    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43899    |\n",
            "|    policy_loss        | 0.0669   |\n",
            "|    std                | 0.0624   |\n",
            "|    value_loss         | 0.00141  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44000    |\n",
            "|    time_elapsed       | 2901     |\n",
            "|    total_timesteps    | 1408000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.24    |\n",
            "|    explained_variance | 0.968    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 43999    |\n",
            "|    policy_loss        | -0.0544  |\n",
            "|    std                | 0.0625   |\n",
            "|    value_loss         | 0.0139   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44100    |\n",
            "|    time_elapsed       | 2906     |\n",
            "|    total_timesteps    | 1411200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.61    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44099    |\n",
            "|    policy_loss        | 0.305    |\n",
            "|    std                | 0.0626   |\n",
            "|    value_loss         | 0.00711  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44200    |\n",
            "|    time_elapsed       | 2914     |\n",
            "|    total_timesteps    | 1414400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.51    |\n",
            "|    explained_variance | 0.899    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44199    |\n",
            "|    policy_loss        | -0.0558  |\n",
            "|    std                | 0.0625   |\n",
            "|    value_loss         | 0.000444 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44300    |\n",
            "|    time_elapsed       | 2919     |\n",
            "|    total_timesteps    | 1417600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.52    |\n",
            "|    explained_variance | 0.94     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44299    |\n",
            "|    policy_loss        | -0.14    |\n",
            "|    std                | 0.0626   |\n",
            "|    value_loss         | 0.0063   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44400    |\n",
            "|    time_elapsed       | 2925     |\n",
            "|    total_timesteps    | 1420800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.45    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44399    |\n",
            "|    policy_loss        | -0.0191  |\n",
            "|    std                | 0.0626   |\n",
            "|    value_loss         | 0.00233  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.63e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44500    |\n",
            "|    time_elapsed       | 2932     |\n",
            "|    total_timesteps    | 1424000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.48    |\n",
            "|    explained_variance | 0.976    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44499    |\n",
            "|    policy_loss        | 0.00958  |\n",
            "|    std                | 0.0625   |\n",
            "|    value_loss         | 0.00357  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.64e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44600    |\n",
            "|    time_elapsed       | 2938     |\n",
            "|    total_timesteps    | 1427200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.76    |\n",
            "|    explained_variance | 0.972    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44599    |\n",
            "|    policy_loss        | -0.175   |\n",
            "|    std                | 0.0628   |\n",
            "|    value_loss         | 0.00174  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.64e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44700    |\n",
            "|    time_elapsed       | 2945     |\n",
            "|    total_timesteps    | 1430400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.51    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44699    |\n",
            "|    policy_loss        | 0.0215   |\n",
            "|    std                | 0.063    |\n",
            "|    value_loss         | 0.00381  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.64e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44800    |\n",
            "|    time_elapsed       | 2951     |\n",
            "|    total_timesteps    | 1433600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.24    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44799    |\n",
            "|    policy_loss        | 0.313    |\n",
            "|    std                | 0.0628   |\n",
            "|    value_loss         | 0.0186   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.64e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 44900    |\n",
            "|    time_elapsed       | 2957     |\n",
            "|    total_timesteps    | 1436800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.52    |\n",
            "|    explained_variance | 0.989    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44899    |\n",
            "|    policy_loss        | 0.212    |\n",
            "|    std                | 0.0627   |\n",
            "|    value_loss         | 0.0026   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.64e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 45000    |\n",
            "|    time_elapsed       | 2964     |\n",
            "|    total_timesteps    | 1440000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.41    |\n",
            "|    explained_variance | 0.974    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 44999    |\n",
            "|    policy_loss        | 0.00726  |\n",
            "|    std                | 0.0622   |\n",
            "|    value_loss         | 0.00229  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 45100    |\n",
            "|    time_elapsed       | 2969     |\n",
            "|    total_timesteps    | 1443200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.42    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45099    |\n",
            "|    policy_loss        | -0.0575  |\n",
            "|    std                | 0.0622   |\n",
            "|    value_loss         | 0.00113  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 45200    |\n",
            "|    time_elapsed       | 2977     |\n",
            "|    total_timesteps    | 1446400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.51    |\n",
            "|    explained_variance | 0.988    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45199    |\n",
            "|    policy_loss        | 0.0571   |\n",
            "|    std                | 0.0622   |\n",
            "|    value_loss         | 0.00296  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 45300    |\n",
            "|    time_elapsed       | 2982     |\n",
            "|    total_timesteps    | 1449600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.31    |\n",
            "|    explained_variance | 0.98     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45299    |\n",
            "|    policy_loss        | 0.0624   |\n",
            "|    std                | 0.0623   |\n",
            "|    value_loss         | 0.000579 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 45400    |\n",
            "|    time_elapsed       | 2989     |\n",
            "|    total_timesteps    | 1452800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.51    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45399    |\n",
            "|    policy_loss        | 0.00855  |\n",
            "|    std                | 0.062    |\n",
            "|    value_loss         | 0.00117  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 45500    |\n",
            "|    time_elapsed       | 2995     |\n",
            "|    total_timesteps    | 1456000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.31    |\n",
            "|    explained_variance | 0.988    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45499    |\n",
            "|    policy_loss        | -0.0101  |\n",
            "|    std                | 0.0618   |\n",
            "|    value_loss         | 0.000646 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 45600    |\n",
            "|    time_elapsed       | 3001     |\n",
            "|    total_timesteps    | 1459200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.42    |\n",
            "|    explained_variance | 0.963    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45599    |\n",
            "|    policy_loss        | 0.0526   |\n",
            "|    std                | 0.0618   |\n",
            "|    value_loss         | 0.000758 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.7e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 45700    |\n",
            "|    time_elapsed       | 3008     |\n",
            "|    total_timesteps    | 1462400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.08    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45699    |\n",
            "|    policy_loss        | 0.0127   |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.0027   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.71e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 45800    |\n",
            "|    time_elapsed       | 3016     |\n",
            "|    total_timesteps    | 1465600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.14    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45799    |\n",
            "|    policy_loss        | 0.0366   |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.000914 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.71e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 45900    |\n",
            "|    time_elapsed       | 3023     |\n",
            "|    total_timesteps    | 1468800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.87    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45899    |\n",
            "|    policy_loss        | 0.085    |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.00115  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.7e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 46000    |\n",
            "|    time_elapsed       | 3029     |\n",
            "|    total_timesteps    | 1472000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.44    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 45999    |\n",
            "|    policy_loss        | -0.0962  |\n",
            "|    std                | 0.0614   |\n",
            "|    value_loss         | 0.00208  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.69e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 46100    |\n",
            "|    time_elapsed       | 3036     |\n",
            "|    total_timesteps    | 1475200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.31    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46099    |\n",
            "|    policy_loss        | 0.0298   |\n",
            "|    std                | 0.0614   |\n",
            "|    value_loss         | 0.00358  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 46200    |\n",
            "|    time_elapsed       | 3042     |\n",
            "|    total_timesteps    | 1478400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.42    |\n",
            "|    explained_variance | 0.972    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46199    |\n",
            "|    policy_loss        | 0.0335   |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.000751 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 46300    |\n",
            "|    time_elapsed       | 3048     |\n",
            "|    total_timesteps    | 1481600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.51    |\n",
            "|    explained_variance | 0.975    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46299    |\n",
            "|    policy_loss        | 0.106    |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.0229   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.63e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 485      |\n",
            "|    iterations         | 46400    |\n",
            "|    time_elapsed       | 3055     |\n",
            "|    total_timesteps    | 1484800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.42    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46399    |\n",
            "|    policy_loss        | -0.0459  |\n",
            "|    std                | 0.0611   |\n",
            "|    value_loss         | 0.000933 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.62e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 46500    |\n",
            "|    time_elapsed       | 3060     |\n",
            "|    total_timesteps    | 1488000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.49    |\n",
            "|    explained_variance | 0.979    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46499    |\n",
            "|    policy_loss        | 0.0538   |\n",
            "|    std                | 0.0613   |\n",
            "|    value_loss         | 0.00241  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.61e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 46600    |\n",
            "|    time_elapsed       | 3068     |\n",
            "|    total_timesteps    | 1491200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.89    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46599    |\n",
            "|    policy_loss        | -0.0898  |\n",
            "|    std                | 0.0613   |\n",
            "|    value_loss         | 0.0033   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.58e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 46700    |\n",
            "|    time_elapsed       | 3073     |\n",
            "|    total_timesteps    | 1494400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.4     |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46699    |\n",
            "|    policy_loss        | 0.15     |\n",
            "|    std                | 0.0614   |\n",
            "|    value_loss         | 0.0022   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.58e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 46800    |\n",
            "|    time_elapsed       | 3080     |\n",
            "|    total_timesteps    | 1497600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.15    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46799    |\n",
            "|    policy_loss        | 0.137    |\n",
            "|    std                | 0.0614   |\n",
            "|    value_loss         | 0.00226  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.57e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 46900    |\n",
            "|    time_elapsed       | 3086     |\n",
            "|    total_timesteps    | 1500800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.41    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46899    |\n",
            "|    policy_loss        | 0.0677   |\n",
            "|    std                | 0.0614   |\n",
            "|    value_loss         | 0.00479  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.57e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47000    |\n",
            "|    time_elapsed       | 3092     |\n",
            "|    total_timesteps    | 1504000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.33    |\n",
            "|    explained_variance | 0.988    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 46999    |\n",
            "|    policy_loss        | -0.279   |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.0135   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.56e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47100    |\n",
            "|    time_elapsed       | 3099     |\n",
            "|    total_timesteps    | 1507200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.18    |\n",
            "|    explained_variance | 0.989    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47099    |\n",
            "|    policy_loss        | -0.148   |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.00428  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.56e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47200    |\n",
            "|    time_elapsed       | 3105     |\n",
            "|    total_timesteps    | 1510400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.23    |\n",
            "|    explained_variance | 0.964    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47199    |\n",
            "|    policy_loss        | -0.0931  |\n",
            "|    std                | 0.0611   |\n",
            "|    value_loss         | 0.00152  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.57e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47300    |\n",
            "|    time_elapsed       | 3113     |\n",
            "|    total_timesteps    | 1513600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.41    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47299    |\n",
            "|    policy_loss        | -0.0731  |\n",
            "|    std                | 0.0614   |\n",
            "|    value_loss         | 0.00105  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.56e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47400    |\n",
            "|    time_elapsed       | 3118     |\n",
            "|    total_timesteps    | 1516800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.36    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47399    |\n",
            "|    policy_loss        | 0.0833   |\n",
            "|    std                | 0.0613   |\n",
            "|    value_loss         | 0.00119  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.56e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47500    |\n",
            "|    time_elapsed       | 3125     |\n",
            "|    total_timesteps    | 1520000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.5     |\n",
            "|    explained_variance | 0.944    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47499    |\n",
            "|    policy_loss        | -0.0165  |\n",
            "|    std                | 0.0613   |\n",
            "|    value_loss         | 0.000232 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.57e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47600    |\n",
            "|    time_elapsed       | 3131     |\n",
            "|    total_timesteps    | 1523200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.09    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47599    |\n",
            "|    policy_loss        | -0.0225  |\n",
            "|    std                | 0.0615   |\n",
            "|    value_loss         | 0.00139  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.57e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47700    |\n",
            "|    time_elapsed       | 3137     |\n",
            "|    total_timesteps    | 1526400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.1     |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47699    |\n",
            "|    policy_loss        | -0.219   |\n",
            "|    std                | 0.0615   |\n",
            "|    value_loss         | 0.00652  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.56e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47800    |\n",
            "|    time_elapsed       | 3144     |\n",
            "|    total_timesteps    | 1529600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.98    |\n",
            "|    explained_variance | 1        |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47799    |\n",
            "|    policy_loss        | -0.0082  |\n",
            "|    std                | 0.0614   |\n",
            "|    value_loss         | 0.00057  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.54e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 47900    |\n",
            "|    time_elapsed       | 3150     |\n",
            "|    total_timesteps    | 1532800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.13    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47899    |\n",
            "|    policy_loss        | 0.0206   |\n",
            "|    std                | 0.0614   |\n",
            "|    value_loss         | 0.000822 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.53e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48000    |\n",
            "|    time_elapsed       | 3157     |\n",
            "|    total_timesteps    | 1536000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.03    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 47999    |\n",
            "|    policy_loss        | 0.0407   |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.000547 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.52e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48100    |\n",
            "|    time_elapsed       | 3163     |\n",
            "|    total_timesteps    | 1539200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.81    |\n",
            "|    explained_variance | 1        |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48099    |\n",
            "|    policy_loss        | 0.0409   |\n",
            "|    std                | 0.061    |\n",
            "|    value_loss         | 0.000695 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.51e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48200    |\n",
            "|    time_elapsed       | 3169     |\n",
            "|    total_timesteps    | 1542400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.22    |\n",
            "|    explained_variance | 0.973    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48199    |\n",
            "|    policy_loss        | -0.0974  |\n",
            "|    std                | 0.0611   |\n",
            "|    value_loss         | 0.00226  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.5e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48300    |\n",
            "|    time_elapsed       | 3176     |\n",
            "|    total_timesteps    | 1545600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.08    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48299    |\n",
            "|    policy_loss        | 0.0546   |\n",
            "|    std                | 0.0609   |\n",
            "|    value_loss         | 0.00188  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.5e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48400    |\n",
            "|    time_elapsed       | 3181     |\n",
            "|    total_timesteps    | 1548800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.01    |\n",
            "|    explained_variance | 0.986    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48399    |\n",
            "|    policy_loss        | 0.0833   |\n",
            "|    std                | 0.0611   |\n",
            "|    value_loss         | 0.00206  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.49e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48500    |\n",
            "|    time_elapsed       | 3189     |\n",
            "|    total_timesteps    | 1552000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.65    |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48499    |\n",
            "|    policy_loss        | -0.0133  |\n",
            "|    std                | 0.0613   |\n",
            "|    value_loss         | 0.00373  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.48e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48600    |\n",
            "|    time_elapsed       | 3194     |\n",
            "|    total_timesteps    | 1555200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.62    |\n",
            "|    explained_variance | 0.969    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48599    |\n",
            "|    policy_loss        | 0.0732   |\n",
            "|    std                | 0.0615   |\n",
            "|    value_loss         | 0.00258  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.45e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48700    |\n",
            "|    time_elapsed       | 3201     |\n",
            "|    total_timesteps    | 1558400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -2.94    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48699    |\n",
            "|    policy_loss        | 0.00716  |\n",
            "|    std                | 0.0616   |\n",
            "|    value_loss         | 0.00032  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.42e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48800    |\n",
            "|    time_elapsed       | 3207     |\n",
            "|    total_timesteps    | 1561600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.52    |\n",
            "|    explained_variance | 0.981    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48799    |\n",
            "|    policy_loss        | -0.0282  |\n",
            "|    std                | 0.0617   |\n",
            "|    value_loss         | 0.000663 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.38e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 48900    |\n",
            "|    time_elapsed       | 3213     |\n",
            "|    total_timesteps    | 1564800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -2.23    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48899    |\n",
            "|    policy_loss        | -0.0546  |\n",
            "|    std                | 0.0618   |\n",
            "|    value_loss         | 0.000957 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49000    |\n",
            "|    time_elapsed       | 3221     |\n",
            "|    total_timesteps    | 1568000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.17    |\n",
            "|    explained_variance | 0.977    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 48999    |\n",
            "|    policy_loss        | 0.0696   |\n",
            "|    std                | 0.0617   |\n",
            "|    value_loss         | 0.00161  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49100    |\n",
            "|    time_elapsed       | 3227     |\n",
            "|    total_timesteps    | 1571200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.78    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49099    |\n",
            "|    policy_loss        | 0.0261   |\n",
            "|    std                | 0.0617   |\n",
            "|    value_loss         | 0.000428 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49200    |\n",
            "|    time_elapsed       | 3236     |\n",
            "|    total_timesteps    | 1574400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.45    |\n",
            "|    explained_variance | 0.989    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49199    |\n",
            "|    policy_loss        | 0.106    |\n",
            "|    std                | 0.0615   |\n",
            "|    value_loss         | 0.00278  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49300    |\n",
            "|    time_elapsed       | 3241     |\n",
            "|    total_timesteps    | 1577600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.33    |\n",
            "|    explained_variance | 0.986    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49299    |\n",
            "|    policy_loss        | -0.18    |\n",
            "|    std                | 0.0618   |\n",
            "|    value_loss         | 0.00525  |\n",
            "------------------------------------\n",
            "-------------------------------------\n",
            "| rollout/              |           |\n",
            "|    ep_len_mean        | 1e+03     |\n",
            "|    ep_rew_mean        | 1.35e+03  |\n",
            "| time/                 |           |\n",
            "|    fps                | 486       |\n",
            "|    iterations         | 49400     |\n",
            "|    time_elapsed       | 3248      |\n",
            "|    total_timesteps    | 1580800   |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -4.08     |\n",
            "|    explained_variance | 0.972     |\n",
            "|    learning_rate      | 0.00096   |\n",
            "|    n_updates          | 49399     |\n",
            "|    policy_loss        | -0.000211 |\n",
            "|    std                | 0.0615    |\n",
            "|    value_loss         | 0.000898  |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49500    |\n",
            "|    time_elapsed       | 3254     |\n",
            "|    total_timesteps    | 1584000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.93    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49499    |\n",
            "|    policy_loss        | 0.0192   |\n",
            "|    std                | 0.0619   |\n",
            "|    value_loss         | 0.00105  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.35e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49600    |\n",
            "|    time_elapsed       | 3260     |\n",
            "|    total_timesteps    | 1587200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.45    |\n",
            "|    explained_variance | 0.77     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49599    |\n",
            "|    policy_loss        | 0.0158   |\n",
            "|    std                | 0.0619   |\n",
            "|    value_loss         | 0.000483 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.36e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49700    |\n",
            "|    time_elapsed       | 3267     |\n",
            "|    total_timesteps    | 1590400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.25    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49699    |\n",
            "|    policy_loss        | 0.0114   |\n",
            "|    std                | 0.062    |\n",
            "|    value_loss         | 0.00164  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.37e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49800    |\n",
            "|    time_elapsed       | 3273     |\n",
            "|    total_timesteps    | 1593600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.44    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49799    |\n",
            "|    policy_loss        | -0.05    |\n",
            "|    std                | 0.062    |\n",
            "|    value_loss         | 0.00109  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.38e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 49900    |\n",
            "|    time_elapsed       | 3280     |\n",
            "|    total_timesteps    | 1596800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.13    |\n",
            "|    explained_variance | 0.991    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49899    |\n",
            "|    policy_loss        | -0.0414  |\n",
            "|    std                | 0.0625   |\n",
            "|    value_loss         | 0.00133  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.38e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 50000    |\n",
            "|    time_elapsed       | 3286     |\n",
            "|    total_timesteps    | 1600000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.35    |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 49999    |\n",
            "|    policy_loss        | -0.116   |\n",
            "|    std                | 0.0625   |\n",
            "|    value_loss         | 0.00433  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.38e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 50100    |\n",
            "|    time_elapsed       | 3292     |\n",
            "|    total_timesteps    | 1603200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.43    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50099    |\n",
            "|    policy_loss        | 0.0358   |\n",
            "|    std                | 0.0624   |\n",
            "|    value_loss         | 0.000287 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.39e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 50200    |\n",
            "|    time_elapsed       | 3299     |\n",
            "|    total_timesteps    | 1606400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.42    |\n",
            "|    explained_variance | 0.972    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50199    |\n",
            "|    policy_loss        | -0.117   |\n",
            "|    std                | 0.0624   |\n",
            "|    value_loss         | 0.00255  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.39e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 50300    |\n",
            "|    time_elapsed       | 3304     |\n",
            "|    total_timesteps    | 1609600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.51    |\n",
            "|    explained_variance | 0.972    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50299    |\n",
            "|    policy_loss        | 0.0695   |\n",
            "|    std                | 0.0629   |\n",
            "|    value_loss         | 0.00209  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.39e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 50400    |\n",
            "|    time_elapsed       | 3312     |\n",
            "|    total_timesteps    | 1612800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.61    |\n",
            "|    explained_variance | 0.986    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50399    |\n",
            "|    policy_loss        | -0.0695  |\n",
            "|    std                | 0.0632   |\n",
            "|    value_loss         | 0.00134  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.4e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 50500    |\n",
            "|    time_elapsed       | 3317     |\n",
            "|    total_timesteps    | 1616000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.59    |\n",
            "|    explained_variance | 0.54     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50499    |\n",
            "|    policy_loss        | 0.023    |\n",
            "|    std                | 0.0629   |\n",
            "|    value_loss         | 0.000147 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.41e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 486      |\n",
            "|    iterations         | 50600    |\n",
            "|    time_elapsed       | 3325     |\n",
            "|    total_timesteps    | 1619200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.51    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50599    |\n",
            "|    policy_loss        | 0.0161   |\n",
            "|    std                | 0.0626   |\n",
            "|    value_loss         | 0.000275 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.41e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 50700    |\n",
            "|    time_elapsed       | 3330     |\n",
            "|    total_timesteps    | 1622400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.44    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50699    |\n",
            "|    policy_loss        | -0.105   |\n",
            "|    std                | 0.0627   |\n",
            "|    value_loss         | 0.00428  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.42e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 50800    |\n",
            "|    time_elapsed       | 3336     |\n",
            "|    total_timesteps    | 1625600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.69    |\n",
            "|    explained_variance | 0.829    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50799    |\n",
            "|    policy_loss        | 0.0071   |\n",
            "|    std                | 0.063    |\n",
            "|    value_loss         | 0.000565 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.44e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 50900    |\n",
            "|    time_elapsed       | 3343     |\n",
            "|    total_timesteps    | 1628800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.37    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50899    |\n",
            "|    policy_loss        | 0.0219   |\n",
            "|    std                | 0.0627   |\n",
            "|    value_loss         | 0.00195  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.46e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51000    |\n",
            "|    time_elapsed       | 3349     |\n",
            "|    total_timesteps    | 1632000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.61    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 50999    |\n",
            "|    policy_loss        | -0.0428  |\n",
            "|    std                | 0.0631   |\n",
            "|    value_loss         | 0.00114  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.48e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51100    |\n",
            "|    time_elapsed       | 3356     |\n",
            "|    total_timesteps    | 1635200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.66    |\n",
            "|    explained_variance | 0.784    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51099    |\n",
            "|    policy_loss        | -0.0136  |\n",
            "|    std                | 0.0631   |\n",
            "|    value_loss         | 0.000234 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.5e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51200    |\n",
            "|    time_elapsed       | 3362     |\n",
            "|    total_timesteps    | 1638400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.18    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51199    |\n",
            "|    policy_loss        | -0.00625 |\n",
            "|    std                | 0.0631   |\n",
            "|    value_loss         | 0.00133  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.51e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51300    |\n",
            "|    time_elapsed       | 3368     |\n",
            "|    total_timesteps    | 1641600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.55    |\n",
            "|    explained_variance | 0.978    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51299    |\n",
            "|    policy_loss        | 0.079    |\n",
            "|    std                | 0.0634   |\n",
            "|    value_loss         | 0.00122  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.51e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51400    |\n",
            "|    time_elapsed       | 3374     |\n",
            "|    total_timesteps    | 1644800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.52    |\n",
            "|    explained_variance | 0.976    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51399    |\n",
            "|    policy_loss        | 0.243    |\n",
            "|    std                | 0.0635   |\n",
            "|    value_loss         | 0.00353  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.52e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51500    |\n",
            "|    time_elapsed       | 3380     |\n",
            "|    total_timesteps    | 1648000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.52    |\n",
            "|    explained_variance | 0.829    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51499    |\n",
            "|    policy_loss        | 0.00712  |\n",
            "|    std                | 0.0633   |\n",
            "|    value_loss         | 0.000435 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.53e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51600    |\n",
            "|    time_elapsed       | 3387     |\n",
            "|    total_timesteps    | 1651200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.48    |\n",
            "|    explained_variance | 0.981    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51599    |\n",
            "|    policy_loss        | 0.0539   |\n",
            "|    std                | 0.063    |\n",
            "|    value_loss         | 0.00101  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.56e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51700    |\n",
            "|    time_elapsed       | 3393     |\n",
            "|    total_timesteps    | 1654400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.57    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51699    |\n",
            "|    policy_loss        | 0.00173  |\n",
            "|    std                | 0.0628   |\n",
            "|    value_loss         | 0.00198  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.58e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51800    |\n",
            "|    time_elapsed       | 3400     |\n",
            "|    total_timesteps    | 1657600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.6     |\n",
            "|    explained_variance | 0.991    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51799    |\n",
            "|    policy_loss        | 0.118    |\n",
            "|    std                | 0.0629   |\n",
            "|    value_loss         | 0.0018   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.61e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 51900    |\n",
            "|    time_elapsed       | 3406     |\n",
            "|    total_timesteps    | 1660800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.31    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51899    |\n",
            "|    policy_loss        | -0.0761  |\n",
            "|    std                | 0.0628   |\n",
            "|    value_loss         | 0.000682 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.64e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52000    |\n",
            "|    time_elapsed       | 3411     |\n",
            "|    total_timesteps    | 1664000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.26    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 51999    |\n",
            "|    policy_loss        | 0.103    |\n",
            "|    std                | 0.0627   |\n",
            "|    value_loss         | 0.00176  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52100    |\n",
            "|    time_elapsed       | 3419     |\n",
            "|    total_timesteps    | 1667200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.49    |\n",
            "|    explained_variance | 0.945    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52099    |\n",
            "|    policy_loss        | -0.0392  |\n",
            "|    std                | 0.0627   |\n",
            "|    value_loss         | 0.000474 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.7e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52200    |\n",
            "|    time_elapsed       | 3424     |\n",
            "|    total_timesteps    | 1670400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.42    |\n",
            "|    explained_variance | 0.92     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52199    |\n",
            "|    policy_loss        | 0.0109   |\n",
            "|    std                | 0.0627   |\n",
            "|    value_loss         | 0.000394 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.72e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52300    |\n",
            "|    time_elapsed       | 3431     |\n",
            "|    total_timesteps    | 1673600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.25    |\n",
            "|    explained_variance | 0.942    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52299    |\n",
            "|    policy_loss        | -0.0385  |\n",
            "|    std                | 0.0629   |\n",
            "|    value_loss         | 0.000462 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.75e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52400    |\n",
            "|    time_elapsed       | 3437     |\n",
            "|    total_timesteps    | 1676800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.34    |\n",
            "|    explained_variance | 0.977    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52399    |\n",
            "|    policy_loss        | -0.0175  |\n",
            "|    std                | 0.0631   |\n",
            "|    value_loss         | 0.00375  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52500    |\n",
            "|    time_elapsed       | 3444     |\n",
            "|    total_timesteps    | 1680000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.33    |\n",
            "|    explained_variance | 0.953    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52499    |\n",
            "|    policy_loss        | -0.0203  |\n",
            "|    std                | 0.0626   |\n",
            "|    value_loss         | 0.00119  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.8e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52600    |\n",
            "|    time_elapsed       | 3452     |\n",
            "|    total_timesteps    | 1683200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.34    |\n",
            "|    explained_variance | 0.835    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52599    |\n",
            "|    policy_loss        | 0.0354   |\n",
            "|    std                | 0.0624   |\n",
            "|    value_loss         | 0.000314 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.84e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52700    |\n",
            "|    time_elapsed       | 3457     |\n",
            "|    total_timesteps    | 1686400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.27    |\n",
            "|    explained_variance | 0.959    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52699    |\n",
            "|    policy_loss        | -0.033   |\n",
            "|    std                | 0.0618   |\n",
            "|    value_loss         | 0.000857 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.84e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52800    |\n",
            "|    time_elapsed       | 3465     |\n",
            "|    total_timesteps    | 1689600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.05    |\n",
            "|    explained_variance | 0.976    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52799    |\n",
            "|    policy_loss        | -0.0534  |\n",
            "|    std                | 0.0617   |\n",
            "|    value_loss         | 0.00728  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.85e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 52900    |\n",
            "|    time_elapsed       | 3470     |\n",
            "|    total_timesteps    | 1692800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.3     |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52899    |\n",
            "|    policy_loss        | -0.112   |\n",
            "|    std                | 0.0618   |\n",
            "|    value_loss         | 0.00155  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.86e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 53000    |\n",
            "|    time_elapsed       | 3477     |\n",
            "|    total_timesteps    | 1696000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.2     |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 52999    |\n",
            "|    policy_loss        | 0.23     |\n",
            "|    std                | 0.0616   |\n",
            "|    value_loss         | 0.0133   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.86e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 53100    |\n",
            "|    time_elapsed       | 3483     |\n",
            "|    total_timesteps    | 1699200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.05    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53099    |\n",
            "|    policy_loss        | -0.336   |\n",
            "|    std                | 0.0613   |\n",
            "|    value_loss         | 0.0139   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.84e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 53200    |\n",
            "|    time_elapsed       | 3489     |\n",
            "|    total_timesteps    | 1702400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.41    |\n",
            "|    explained_variance | 0.971    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53199    |\n",
            "|    policy_loss        | 0.0162   |\n",
            "|    std                | 0.0615   |\n",
            "|    value_loss         | 0.00245  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.83e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 53300    |\n",
            "|    time_elapsed       | 3496     |\n",
            "|    total_timesteps    | 1705600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.18    |\n",
            "|    explained_variance | 0.981    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53299    |\n",
            "|    policy_loss        | 0.0435   |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.00264  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.82e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 53400    |\n",
            "|    time_elapsed       | 3502     |\n",
            "|    total_timesteps    | 1708800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.21    |\n",
            "|    explained_variance | 0.961    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53399    |\n",
            "|    policy_loss        | -0.121   |\n",
            "|    std                | 0.0611   |\n",
            "|    value_loss         | 0.00489  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.82e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 53500    |\n",
            "|    time_elapsed       | 3509     |\n",
            "|    total_timesteps    | 1712000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.2     |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53499    |\n",
            "|    policy_loss        | -0.0497  |\n",
            "|    std                | 0.0612   |\n",
            "|    value_loss         | 0.0027   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.82e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 53600    |\n",
            "|    time_elapsed       | 3515     |\n",
            "|    total_timesteps    | 1715200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.34    |\n",
            "|    explained_variance | 0.991    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53599    |\n",
            "|    policy_loss        | 0.0155   |\n",
            "|    std                | 0.0609   |\n",
            "|    value_loss         | 0.0021   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.82e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 53700    |\n",
            "|    time_elapsed       | 3520     |\n",
            "|    total_timesteps    | 1718400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.1     |\n",
            "|    explained_variance | 0.96     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53699    |\n",
            "|    policy_loss        | 0.459    |\n",
            "|    std                | 0.0611   |\n",
            "|    value_loss         | 0.012    |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.82e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 53800    |\n",
            "|    time_elapsed       | 3528     |\n",
            "|    total_timesteps    | 1721600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.37    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53799    |\n",
            "|    policy_loss        | 0.00287  |\n",
            "|    std                | 0.0609   |\n",
            "|    value_loss         | 0.000546 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.82e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 53900    |\n",
            "|    time_elapsed       | 3533     |\n",
            "|    total_timesteps    | 1724800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.06    |\n",
            "|    explained_variance | 0.999    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53899    |\n",
            "|    policy_loss        | 0.0619   |\n",
            "|    std                | 0.0608   |\n",
            "|    value_loss         | 0.000698 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.82e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 487      |\n",
            "|    iterations         | 54000    |\n",
            "|    time_elapsed       | 3541     |\n",
            "|    total_timesteps    | 1728000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.09    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 53999    |\n",
            "|    policy_loss        | 0.00511  |\n",
            "|    std                | 0.0606   |\n",
            "|    value_loss         | 0.00241  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.82e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54100    |\n",
            "|    time_elapsed       | 3546     |\n",
            "|    total_timesteps    | 1731200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.26    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54099    |\n",
            "|    policy_loss        | -0.15    |\n",
            "|    std                | 0.0605   |\n",
            "|    value_loss         | 0.00435  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.8e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54200    |\n",
            "|    time_elapsed       | 3552     |\n",
            "|    total_timesteps    | 1734400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.27    |\n",
            "|    explained_variance | 0.987    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54199    |\n",
            "|    policy_loss        | -0.223   |\n",
            "|    std                | 0.0604   |\n",
            "|    value_loss         | 0.00903  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54300    |\n",
            "|    time_elapsed       | 3559     |\n",
            "|    total_timesteps    | 1737600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.23    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54299    |\n",
            "|    policy_loss        | -0.129   |\n",
            "|    std                | 0.0605   |\n",
            "|    value_loss         | 0.00184  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54400    |\n",
            "|    time_elapsed       | 3565     |\n",
            "|    total_timesteps    | 1740800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.81    |\n",
            "|    explained_variance | 0.979    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54399    |\n",
            "|    policy_loss        | -0.0364  |\n",
            "|    std                | 0.0604   |\n",
            "|    value_loss         | 0.00129  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54500    |\n",
            "|    time_elapsed       | 3572     |\n",
            "|    total_timesteps    | 1744000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.14    |\n",
            "|    explained_variance | 0.938    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54499    |\n",
            "|    policy_loss        | -0.018   |\n",
            "|    std                | 0.0603   |\n",
            "|    value_loss         | 0.000525 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54600    |\n",
            "|    time_elapsed       | 3578     |\n",
            "|    total_timesteps    | 1747200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.2     |\n",
            "|    explained_variance | 0.907    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54599    |\n",
            "|    policy_loss        | -0.0788  |\n",
            "|    std                | 0.06     |\n",
            "|    value_loss         | 0.00103  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54700    |\n",
            "|    time_elapsed       | 3584     |\n",
            "|    total_timesteps    | 1750400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.1     |\n",
            "|    explained_variance | 0.977    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54699    |\n",
            "|    policy_loss        | -0.127   |\n",
            "|    std                | 0.0598   |\n",
            "|    value_loss         | 0.00504  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.77e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54800    |\n",
            "|    time_elapsed       | 3591     |\n",
            "|    total_timesteps    | 1753600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.16    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54799    |\n",
            "|    policy_loss        | 0.0571   |\n",
            "|    std                | 0.0599   |\n",
            "|    value_loss         | 0.00829  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.75e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 54900    |\n",
            "|    time_elapsed       | 3596     |\n",
            "|    total_timesteps    | 1756800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.91    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54899    |\n",
            "|    policy_loss        | 0.0363   |\n",
            "|    std                | 0.0599   |\n",
            "|    value_loss         | 0.00193  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55000    |\n",
            "|    time_elapsed       | 3604     |\n",
            "|    total_timesteps    | 1760000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.29    |\n",
            "|    explained_variance | 0.963    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 54999    |\n",
            "|    policy_loss        | 0.0504   |\n",
            "|    std                | 0.0597   |\n",
            "|    value_loss         | 0.00785  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55100    |\n",
            "|    time_elapsed       | 3610     |\n",
            "|    total_timesteps    | 1763200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.16    |\n",
            "|    explained_variance | 0.957    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55099    |\n",
            "|    policy_loss        | 0.366    |\n",
            "|    std                | 0.0595   |\n",
            "|    value_loss         | 0.0118   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.72e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55200    |\n",
            "|    time_elapsed       | 3617     |\n",
            "|    total_timesteps    | 1766400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.16    |\n",
            "|    explained_variance | 0.982    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55199    |\n",
            "|    policy_loss        | -0.12    |\n",
            "|    std                | 0.0596   |\n",
            "|    value_loss         | 0.012    |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.7e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55300    |\n",
            "|    time_elapsed       | 3623     |\n",
            "|    total_timesteps    | 1769600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4       |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55299    |\n",
            "|    policy_loss        | 0.0369   |\n",
            "|    std                | 0.0596   |\n",
            "|    value_loss         | 0.00521  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55400    |\n",
            "|    time_elapsed       | 3630     |\n",
            "|    total_timesteps    | 1772800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.16    |\n",
            "|    explained_variance | 0.99     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55399    |\n",
            "|    policy_loss        | -0.0758  |\n",
            "|    std                | 0.0599   |\n",
            "|    value_loss         | 0.00395  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55500    |\n",
            "|    time_elapsed       | 3636     |\n",
            "|    total_timesteps    | 1776000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.97    |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55499    |\n",
            "|    policy_loss        | -0.112   |\n",
            "|    std                | 0.0594   |\n",
            "|    value_loss         | 0.00777  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55600    |\n",
            "|    time_elapsed       | 3642     |\n",
            "|    total_timesteps    | 1779200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.05    |\n",
            "|    explained_variance | 0.876    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55599    |\n",
            "|    policy_loss        | 0.196    |\n",
            "|    std                | 0.0592   |\n",
            "|    value_loss         | 0.00459  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55700    |\n",
            "|    time_elapsed       | 3649     |\n",
            "|    total_timesteps    | 1782400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.22    |\n",
            "|    explained_variance | 0.893    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55699    |\n",
            "|    policy_loss        | -0.117   |\n",
            "|    std                | 0.0593   |\n",
            "|    value_loss         | 0.00177  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.65e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55800    |\n",
            "|    time_elapsed       | 3655     |\n",
            "|    total_timesteps    | 1785600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.04    |\n",
            "|    explained_variance | 0.961    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55799    |\n",
            "|    policy_loss        | 0.0365   |\n",
            "|    std                | 0.0594   |\n",
            "|    value_loss         | 0.00128  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 55900    |\n",
            "|    time_elapsed       | 3664     |\n",
            "|    total_timesteps    | 1788800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.08    |\n",
            "|    explained_variance | 0.987    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55899    |\n",
            "|    policy_loss        | -0.0243  |\n",
            "|    std                | 0.0592   |\n",
            "|    value_loss         | 0.00163  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.66e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56000    |\n",
            "|    time_elapsed       | 3670     |\n",
            "|    total_timesteps    | 1792000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.97    |\n",
            "|    explained_variance | 0.915    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 55999    |\n",
            "|    policy_loss        | 0.0391   |\n",
            "|    std                | 0.0591   |\n",
            "|    value_loss         | 0.000638 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.67e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56100    |\n",
            "|    time_elapsed       | 3676     |\n",
            "|    total_timesteps    | 1795200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.19    |\n",
            "|    explained_variance | 0.898    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56099    |\n",
            "|    policy_loss        | -0.0108  |\n",
            "|    std                | 0.0592   |\n",
            "|    value_loss         | 0.00119  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.68e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56200    |\n",
            "|    time_elapsed       | 3683     |\n",
            "|    total_timesteps    | 1798400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.2     |\n",
            "|    explained_variance | 0.973    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56199    |\n",
            "|    policy_loss        | 0.0117   |\n",
            "|    std                | 0.0594   |\n",
            "|    value_loss         | 0.000215 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.72e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56300    |\n",
            "|    time_elapsed       | 3689     |\n",
            "|    total_timesteps    | 1801600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.07    |\n",
            "|    explained_variance | 0.426    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56299    |\n",
            "|    policy_loss        | -0.0239  |\n",
            "|    std                | 0.059    |\n",
            "|    value_loss         | 0.000439 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.74e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56400    |\n",
            "|    time_elapsed       | 3696     |\n",
            "|    total_timesteps    | 1804800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.88    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56399    |\n",
            "|    policy_loss        | -0.00768 |\n",
            "|    std                | 0.0588   |\n",
            "|    value_loss         | 0.000234 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56500    |\n",
            "|    time_elapsed       | 3702     |\n",
            "|    total_timesteps    | 1808000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.91    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56499    |\n",
            "|    policy_loss        | -0.018   |\n",
            "|    std                | 0.0588   |\n",
            "|    value_loss         | 0.000545 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.77e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56600    |\n",
            "|    time_elapsed       | 3709     |\n",
            "|    total_timesteps    | 1811200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.88    |\n",
            "|    explained_variance | 0.941    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56599    |\n",
            "|    policy_loss        | -0.0784  |\n",
            "|    std                | 0.0586   |\n",
            "|    value_loss         | 0.00131  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.77e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56700    |\n",
            "|    time_elapsed       | 3715     |\n",
            "|    total_timesteps    | 1814400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.89    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56699    |\n",
            "|    policy_loss        | -0.0941  |\n",
            "|    std                | 0.0585   |\n",
            "|    value_loss         | 0.00177  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.75e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56800    |\n",
            "|    time_elapsed       | 3721     |\n",
            "|    total_timesteps    | 1817600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.77    |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56799    |\n",
            "|    policy_loss        | 0.283    |\n",
            "|    std                | 0.0588   |\n",
            "|    value_loss         | 0.00776  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.72e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 56900    |\n",
            "|    time_elapsed       | 3728     |\n",
            "|    total_timesteps    | 1820800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.88    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56899    |\n",
            "|    policy_loss        | 0.129    |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.00264  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.7e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57000    |\n",
            "|    time_elapsed       | 3734     |\n",
            "|    total_timesteps    | 1824000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.88    |\n",
            "|    explained_variance | 0.749    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 56999    |\n",
            "|    policy_loss        | 0.0399   |\n",
            "|    std                | 0.0587   |\n",
            "|    value_loss         | 0.00347  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.7e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57100    |\n",
            "|    time_elapsed       | 3741     |\n",
            "|    total_timesteps    | 1827200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.93    |\n",
            "|    explained_variance | 0.956    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57099    |\n",
            "|    policy_loss        | 0.0071   |\n",
            "|    std                | 0.0586   |\n",
            "|    value_loss         | 0.000327 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.71e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57200    |\n",
            "|    time_elapsed       | 3747     |\n",
            "|    total_timesteps    | 1830400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.14    |\n",
            "|    explained_variance | 0.907    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57199    |\n",
            "|    policy_loss        | 0.000709 |\n",
            "|    std                | 0.0587   |\n",
            "|    value_loss         | 0.000849 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.72e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57300    |\n",
            "|    time_elapsed       | 3753     |\n",
            "|    total_timesteps    | 1833600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.69    |\n",
            "|    explained_variance | 1        |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57299    |\n",
            "|    policy_loss        | 0.0063   |\n",
            "|    std                | 0.0588   |\n",
            "|    value_loss         | 0.000274 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57400    |\n",
            "|    time_elapsed       | 3760     |\n",
            "|    total_timesteps    | 1836800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.96    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57399    |\n",
            "|    policy_loss        | -0.0262  |\n",
            "|    std                | 0.0587   |\n",
            "|    value_loss         | 0.000497 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.75e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57500    |\n",
            "|    time_elapsed       | 3766     |\n",
            "|    total_timesteps    | 1840000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.77    |\n",
            "|    explained_variance | 0.97     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57499    |\n",
            "|    policy_loss        | 0.0639   |\n",
            "|    std                | 0.0587   |\n",
            "|    value_loss         | 0.00805  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57600    |\n",
            "|    time_elapsed       | 3773     |\n",
            "|    total_timesteps    | 1843200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.81    |\n",
            "|    explained_variance | 0.893    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57599    |\n",
            "|    policy_loss        | 0.00472  |\n",
            "|    std                | 0.0588   |\n",
            "|    value_loss         | 0.00112  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57700    |\n",
            "|    time_elapsed       | 3779     |\n",
            "|    total_timesteps    | 1846400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.85    |\n",
            "|    explained_variance | 0.86     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57699    |\n",
            "|    policy_loss        | 0.00447  |\n",
            "|    std                | 0.0586   |\n",
            "|    value_loss         | 0.000515 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57800    |\n",
            "|    time_elapsed       | 3786     |\n",
            "|    total_timesteps    | 1849600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.76    |\n",
            "|    explained_variance | 0.922    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57799    |\n",
            "|    policy_loss        | 0.0881   |\n",
            "|    std                | 0.0585   |\n",
            "|    value_loss         | 0.00115  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.77e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 57900    |\n",
            "|    time_elapsed       | 3792     |\n",
            "|    total_timesteps    | 1852800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.98    |\n",
            "|    explained_variance | 0.984    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57899    |\n",
            "|    policy_loss        | -0.0534  |\n",
            "|    std                | 0.0585   |\n",
            "|    value_loss         | 0.00193  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58000    |\n",
            "|    time_elapsed       | 3798     |\n",
            "|    total_timesteps    | 1856000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.89    |\n",
            "|    explained_variance | 0.993    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 57999    |\n",
            "|    policy_loss        | 0.00482  |\n",
            "|    std                | 0.0586   |\n",
            "|    value_loss         | 0.00176  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.8e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58100    |\n",
            "|    time_elapsed       | 3805     |\n",
            "|    total_timesteps    | 1859200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.86    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58099    |\n",
            "|    policy_loss        | 0.554    |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.0388   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.81e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58200    |\n",
            "|    time_elapsed       | 3811     |\n",
            "|    total_timesteps    | 1862400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.29    |\n",
            "|    explained_variance | 0.968    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58199    |\n",
            "|    policy_loss        | 0.0511   |\n",
            "|    std                | 0.0588   |\n",
            "|    value_loss         | 0.0068   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58300    |\n",
            "|    time_elapsed       | 3818     |\n",
            "|    total_timesteps    | 1865600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.7     |\n",
            "|    explained_variance | 0.974    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58299    |\n",
            "|    policy_loss        | -0.329   |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.0136   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58400    |\n",
            "|    time_elapsed       | 3824     |\n",
            "|    total_timesteps    | 1868800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.91    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58399    |\n",
            "|    policy_loss        | 0.092    |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.00106  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58500    |\n",
            "|    time_elapsed       | 3831     |\n",
            "|    total_timesteps    | 1872000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.83    |\n",
            "|    explained_variance | 0.945    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58499    |\n",
            "|    policy_loss        | -0.0131  |\n",
            "|    std                | 0.0588   |\n",
            "|    value_loss         | 0.00077  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58600    |\n",
            "|    time_elapsed       | 3837     |\n",
            "|    total_timesteps    | 1875200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.86    |\n",
            "|    explained_variance | 0.976    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58599    |\n",
            "|    policy_loss        | -0.0393  |\n",
            "|    std                | 0.0591   |\n",
            "|    value_loss         | 0.00222  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.8e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58700    |\n",
            "|    time_elapsed       | 3843     |\n",
            "|    total_timesteps    | 1878400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.88    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58699    |\n",
            "|    policy_loss        | -0.0474  |\n",
            "|    std                | 0.059    |\n",
            "|    value_loss         | 0.00247  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.8e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58800    |\n",
            "|    time_elapsed       | 3850     |\n",
            "|    total_timesteps    | 1881600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.81    |\n",
            "|    explained_variance | 0.986    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58799    |\n",
            "|    policy_loss        | 0.00446  |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.000656 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 58900    |\n",
            "|    time_elapsed       | 3856     |\n",
            "|    total_timesteps    | 1884800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.93    |\n",
            "|    explained_variance | 0.988    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58899    |\n",
            "|    policy_loss        | -0.36    |\n",
            "|    std                | 0.0585   |\n",
            "|    value_loss         | 0.00831  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59000    |\n",
            "|    time_elapsed       | 3863     |\n",
            "|    total_timesteps    | 1888000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.9     |\n",
            "|    explained_variance | 0.632    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 58999    |\n",
            "|    policy_loss        | 0.077    |\n",
            "|    std                | 0.058    |\n",
            "|    value_loss         | 0.00205  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59100    |\n",
            "|    time_elapsed       | 3869     |\n",
            "|    total_timesteps    | 1891200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.84    |\n",
            "|    explained_variance | 0.955    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59099    |\n",
            "|    policy_loss        | 0.0209   |\n",
            "|    std                | 0.0577   |\n",
            "|    value_loss         | 0.000342 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59200    |\n",
            "|    time_elapsed       | 3875     |\n",
            "|    total_timesteps    | 1894400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.84    |\n",
            "|    explained_variance | 0.691    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59199    |\n",
            "|    policy_loss        | -0.0447  |\n",
            "|    std                | 0.0578   |\n",
            "|    value_loss         | 0.00152  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59300    |\n",
            "|    time_elapsed       | 3884     |\n",
            "|    total_timesteps    | 1897600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.42    |\n",
            "|    explained_variance | 0.981    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59299    |\n",
            "|    policy_loss        | -0.152   |\n",
            "|    std                | 0.0577   |\n",
            "|    value_loss         | 0.0186   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.75e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59400    |\n",
            "|    time_elapsed       | 3890     |\n",
            "|    total_timesteps    | 1900800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.87    |\n",
            "|    explained_variance | 0.934    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59399    |\n",
            "|    policy_loss        | 0.0947   |\n",
            "|    std                | 0.0576   |\n",
            "|    value_loss         | 0.0029   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.74e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59500    |\n",
            "|    time_elapsed       | 3897     |\n",
            "|    total_timesteps    | 1904000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.91    |\n",
            "|    explained_variance | 0.952    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59499    |\n",
            "|    policy_loss        | 0.0809   |\n",
            "|    std                | 0.0575   |\n",
            "|    value_loss         | 0.00085  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.74e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59600    |\n",
            "|    time_elapsed       | 3903     |\n",
            "|    total_timesteps    | 1907200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.72    |\n",
            "|    explained_variance | 0.958    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59599    |\n",
            "|    policy_loss        | -0.0604  |\n",
            "|    std                | 0.0577   |\n",
            "|    value_loss         | 0.0059   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59700    |\n",
            "|    time_elapsed       | 3910     |\n",
            "|    total_timesteps    | 1910400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.66    |\n",
            "|    explained_variance | 0.863    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59699    |\n",
            "|    policy_loss        | 0.109    |\n",
            "|    std                | 0.0576   |\n",
            "|    value_loss         | 0.00264  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.72e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59800    |\n",
            "|    time_elapsed       | 3916     |\n",
            "|    total_timesteps    | 1913600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.83    |\n",
            "|    explained_variance | 0.633    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59799    |\n",
            "|    policy_loss        | 0.015    |\n",
            "|    std                | 0.058    |\n",
            "|    value_loss         | 0.00131  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.73e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 59900    |\n",
            "|    time_elapsed       | 3922     |\n",
            "|    total_timesteps    | 1916800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.85    |\n",
            "|    explained_variance | 0.995    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59899    |\n",
            "|    policy_loss        | 0.058    |\n",
            "|    std                | 0.0583   |\n",
            "|    value_loss         | 0.00191  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.76e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60000    |\n",
            "|    time_elapsed       | 3929     |\n",
            "|    total_timesteps    | 1920000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.84    |\n",
            "|    explained_variance | 0.87     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 59999    |\n",
            "|    policy_loss        | -0.0622  |\n",
            "|    std                | 0.0582   |\n",
            "|    value_loss         | 0.000708 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60100    |\n",
            "|    time_elapsed       | 3934     |\n",
            "|    total_timesteps    | 1923200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.56    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60099    |\n",
            "|    policy_loss        | 0.116    |\n",
            "|    std                | 0.0578   |\n",
            "|    value_loss         | 0.0025   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60200    |\n",
            "|    time_elapsed       | 3942     |\n",
            "|    total_timesteps    | 1926400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.79    |\n",
            "|    explained_variance | 0.992    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60199    |\n",
            "|    policy_loss        | 0.155    |\n",
            "|    std                | 0.0579   |\n",
            "|    value_loss         | 0.00518  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60300    |\n",
            "|    time_elapsed       | 3947     |\n",
            "|    total_timesteps    | 1929600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.84    |\n",
            "|    explained_variance | 0.902    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60299    |\n",
            "|    policy_loss        | -0.0972  |\n",
            "|    std                | 0.058    |\n",
            "|    value_loss         | 0.00443  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60400    |\n",
            "|    time_elapsed       | 3954     |\n",
            "|    total_timesteps    | 1932800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.73    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60399    |\n",
            "|    policy_loss        | -0.00538 |\n",
            "|    std                | 0.0581   |\n",
            "|    value_loss         | 0.00144  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.8e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60500    |\n",
            "|    time_elapsed       | 3961     |\n",
            "|    total_timesteps    | 1936000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.69    |\n",
            "|    explained_variance | 0.86     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60499    |\n",
            "|    policy_loss        | -0.07    |\n",
            "|    std                | 0.0578   |\n",
            "|    value_loss         | 0.00152  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.8e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60600    |\n",
            "|    time_elapsed       | 3966     |\n",
            "|    total_timesteps    | 1939200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.73    |\n",
            "|    explained_variance | 0.994    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60599    |\n",
            "|    policy_loss        | -0.0323  |\n",
            "|    std                | 0.058    |\n",
            "|    value_loss         | 0.000435 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.79e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60700    |\n",
            "|    time_elapsed       | 3974     |\n",
            "|    total_timesteps    | 1942400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.78    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60699    |\n",
            "|    policy_loss        | 0.0484   |\n",
            "|    std                | 0.0583   |\n",
            "|    value_loss         | 0.00102  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60800    |\n",
            "|    time_elapsed       | 3979     |\n",
            "|    total_timesteps    | 1945600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.72    |\n",
            "|    explained_variance | 0.982    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60799    |\n",
            "|    policy_loss        | -0.0212  |\n",
            "|    std                | 0.0587   |\n",
            "|    value_loss         | 0.0015   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 60900    |\n",
            "|    time_elapsed       | 3987     |\n",
            "|    total_timesteps    | 1948800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.76    |\n",
            "|    explained_variance | 0.929    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60899    |\n",
            "|    policy_loss        | 0.0104   |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.00784  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.77e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 61000    |\n",
            "|    time_elapsed       | 3992     |\n",
            "|    total_timesteps    | 1952000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.61    |\n",
            "|    explained_variance | 0.977    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 60999    |\n",
            "|    policy_loss        | 0.0203   |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.00401  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.77e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 61100    |\n",
            "|    time_elapsed       | 3999     |\n",
            "|    total_timesteps    | 1955200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.76    |\n",
            "|    explained_variance | 0.778    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61099    |\n",
            "|    policy_loss        | -0.165   |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.00828  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.78e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 61200    |\n",
            "|    time_elapsed       | 4005     |\n",
            "|    total_timesteps    | 1958400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.65    |\n",
            "|    explained_variance | 0.989    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61199    |\n",
            "|    policy_loss        | -0.0135  |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.00116  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.8e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 61300    |\n",
            "|    time_elapsed       | 4011     |\n",
            "|    total_timesteps    | 1961600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.6     |\n",
            "|    explained_variance | 0.983    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61299    |\n",
            "|    policy_loss        | -0.00821 |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.0014   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.84e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 61400    |\n",
            "|    time_elapsed       | 4018     |\n",
            "|    total_timesteps    | 1964800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.84    |\n",
            "|    explained_variance | 0.99     |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61399    |\n",
            "|    policy_loss        | -0.096   |\n",
            "|    std                | 0.0589   |\n",
            "|    value_loss         | 0.00181  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.87e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 61500    |\n",
            "|    time_elapsed       | 4024     |\n",
            "|    total_timesteps    | 1968000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.62    |\n",
            "|    explained_variance | 0.951    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61499    |\n",
            "|    policy_loss        | -0.0396  |\n",
            "|    std                | 0.0582   |\n",
            "|    value_loss         | 0.00525  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.88e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 61600    |\n",
            "|    time_elapsed       | 4031     |\n",
            "|    total_timesteps    | 1971200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.76    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61599    |\n",
            "|    policy_loss        | -0.0851  |\n",
            "|    std                | 0.0582   |\n",
            "|    value_loss         | 0.0028   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.88e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 488      |\n",
            "|    iterations         | 61700    |\n",
            "|    time_elapsed       | 4037     |\n",
            "|    total_timesteps    | 1974400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.57    |\n",
            "|    explained_variance | 0.996    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61699    |\n",
            "|    policy_loss        | 0.0247   |\n",
            "|    std                | 0.0585   |\n",
            "|    value_loss         | 0.00171  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.88e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 61800    |\n",
            "|    time_elapsed       | 4043     |\n",
            "|    total_timesteps    | 1977600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.66    |\n",
            "|    explained_variance | 0.982    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61799    |\n",
            "|    policy_loss        | 0.0757   |\n",
            "|    std                | 0.0586   |\n",
            "|    value_loss         | 0.00163  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.89e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 61900    |\n",
            "|    time_elapsed       | 4050     |\n",
            "|    total_timesteps    | 1980800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.78    |\n",
            "|    explained_variance | 0.997    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61899    |\n",
            "|    policy_loss        | -0.0383  |\n",
            "|    std                | 0.0587   |\n",
            "|    value_loss         | 0.000743 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.89e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 62000    |\n",
            "|    time_elapsed       | 4056     |\n",
            "|    total_timesteps    | 1984000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.83    |\n",
            "|    explained_variance | 0.987    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 61999    |\n",
            "|    policy_loss        | -0.0475  |\n",
            "|    std                | 0.059    |\n",
            "|    value_loss         | 0.000935 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.9e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 62100    |\n",
            "|    time_elapsed       | 4063     |\n",
            "|    total_timesteps    | 1987200  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.71    |\n",
            "|    explained_variance | 0.998    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 62099    |\n",
            "|    policy_loss        | -0.0379  |\n",
            "|    std                | 0.0592   |\n",
            "|    value_loss         | 0.00667  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.9e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 62200    |\n",
            "|    time_elapsed       | 4069     |\n",
            "|    total_timesteps    | 1990400  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.88    |\n",
            "|    explained_variance | 0.956    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 62199    |\n",
            "|    policy_loss        | 0.0148   |\n",
            "|    std                | 0.0592   |\n",
            "|    value_loss         | 0.000666 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.9e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 62300    |\n",
            "|    time_elapsed       | 4075     |\n",
            "|    total_timesteps    | 1993600  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.88    |\n",
            "|    explained_variance | 0.937    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 62299    |\n",
            "|    policy_loss        | 0.00297  |\n",
            "|    std                | 0.0593   |\n",
            "|    value_loss         | 0.00126  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.9e+03  |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 62400    |\n",
            "|    time_elapsed       | 4082     |\n",
            "|    total_timesteps    | 1996800  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.7     |\n",
            "|    explained_variance | 0.972    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 62399    |\n",
            "|    policy_loss        | -0.129   |\n",
            "|    std                | 0.0599   |\n",
            "|    value_loss         | 0.00441  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 1e+03    |\n",
            "|    ep_rew_mean        | 1.93e+03 |\n",
            "| time/                 |          |\n",
            "|    fps                | 489      |\n",
            "|    iterations         | 62500    |\n",
            "|    time_elapsed       | 4087     |\n",
            "|    total_timesteps    | 2000000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.97    |\n",
            "|    explained_variance | 0.729    |\n",
            "|    learning_rate      | 0.00096  |\n",
            "|    n_updates          | 62499    |\n",
            "|    policy_loss        | 0.0802   |\n",
            "|    std                | 0.0596   |\n",
            "|    value_loss         | 0.00409  |\n",
            "------------------------------------\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.a2c.a2c.A2C at 0x7fa8bdf61130>"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Save the model and  VecNormalize statistics when saving the agent\n",
        "model.save(\"a2c-AntBulletEnv-v0\")\n",
        "env.save(\"vec_normalize.pkl\")"
      ],
      "metadata": {
        "id": "MfYtjj19cKFr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluate the agent 📈\n",
        "- Now that's our  agent is trained, we need to **check its performance**.\n",
        "- Stable-Baselines3 provides a method to do that: `evaluate_policy`\n",
        "- In my case, I got a mean reward of `2371.90 +/- 16.50`"
      ],
      "metadata": {
        "id": "01M9GCd32Ig-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize\n",
        "\n",
        "# Load the saved statistics\n",
        "eval_env = DummyVecEnv([lambda: gym.make(\"AntBulletEnv-v0\")])\n",
        "eval_env = VecNormalize.load(\"vec_normalize.pkl\", eval_env)\n",
        "\n",
        "#  do not update them at test time\n",
        "eval_env.training = False\n",
        "# reward normalization is not needed at test time\n",
        "eval_env.norm_reward = False\n",
        "\n",
        "# Load the agent\n",
        "model = A2C.load(\"a2c-AntBulletEnv-v0\")\n",
        "\n",
        "mean_reward, std_reward = evaluate_policy(model, eval_env)\n",
        "\n",
        "print(f\"Mean reward = {mean_reward:.2f} +/- {std_reward:.2f}\")"
      ],
      "metadata": {
        "id": "liirTVoDkHq3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "08d0032e-b46a-4df6-82a7-95dc78f27d2a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mean reward = 1888.63 +/- 282.14\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Publish your trained model on the Hub 🔥\n",
        "Now that we saw we got good results after the training, we can publish our trained model on the Hub with one line of code.\n",
        "\n",
        "📚 The libraries documentation 👉 https://github.com/huggingface/huggingface_sb3/tree/main#hugging-face--x-stable-baselines3-v20\n",
        "\n",
        "Here's an example of a Model Card (with a PyBullet environment):\n",
        "\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/modelcardpybullet.png\" alt=\"Model Card Pybullet\"/>"
      ],
      "metadata": {
        "id": "44L9LVQaavR8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "By using `package_to_hub`, as we already mentionned in the former units, **you evaluate, record a replay, generate a model card of your agent and push it to the hub**.\n",
        "\n",
        "This way:\n",
        "- You can **showcase our work** 🔥\n",
        "- You can **visualize your agent playing** 👀\n",
        "- You can **share with the community an agent that others can use** 💾\n",
        "- You can **access a leaderboard 🏆 to see how well your agent is performing compared to your classmates** 👉 https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard\n"
      ],
      "metadata": {
        "id": "MkMk99m8bgaQ"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JquRrWytA6eo"
      },
      "source": [
        "To be able to share your model with the community there are three more steps to follow:\n",
        "\n",
        "1️⃣ (If it's not already done) create an account to HF ➡ https://huggingface.co/join\n",
        "\n",
        "2️⃣ Sign in and then, you need to store your authentication token from the Hugging Face website.\n",
        "- Create a new token (https://huggingface.co/settings/tokens) **with write role**\n",
        "\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg\" alt=\"Create HF Token\">\n",
        "\n",
        "- Copy the token \n",
        "- Run the cell below and paste the token"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GZiFBBlzxzxY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 387,
          "referenced_widgets": [
            "8daa82d0b3d04077a2da05e1c1b77320",
            "6d5f3cee6ee2477ba50e5a2fcb40f658",
            "dafe16625f51423e93184de71cd0d3b1",
            "ff516564269f44f2bc24b57385a38f28",
            "6c1f4085057a4751a1d10ce2be4aba6a",
            "7b60a431d4aa46e6a439786be053ddd5",
            "34560688d0954fbcbfac6ef53b4a9158",
            "a54cfeed81e74dd5bdb42224bec7f106",
            "7e0d0e5d3262438eafaafa1c53745eb4",
            "0b73207b81c34e5394fa2c9dfe8787cd",
            "b5fd7b531aeb43ac917403aacd9b9d06",
            "ea5047dd899f4ef4a0319e479dbdeaef",
            "f867edb6e9674597a97f7bdf2394b004",
            "6d7f165380b040ae934d805db9245825",
            "ec469f3d62714626ac6d525061da01c4",
            "4b05518858f14b9d955af42e4f9088b9",
            "560749b5170149c39c73d33dc0c50f0f"
          ]
        },
        "outputId": "96a3a7b6-fd14-4e46-e294-e39c5550d56e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Token is valid.\n",
            "Your token has been saved in your configured git credential helpers (store).\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ],
      "source": [
        "notebook_login()\n",
        "!git config --global credential.helper store"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_tsf2uv0g_4p"
      },
      "source": [
        "If you don't want to use a Google Colab or a Jupyter Notebook, you need to use this command instead: `huggingface-cli login`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FGNh9VsZok0i"
      },
      "source": [
        "3️⃣ We're now ready to push our trained agent to the 🤗 Hub 🔥 using `package_to_hub()` function"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "package_to_hub(\n",
        "    model=model,\n",
        "    model_name=f\"a2c-{env_id}\",\n",
        "    model_architecture=\"A2C\",\n",
        "    env_id=env_id,\n",
        "    eval_env=eval_env,\n",
        "    repo_id=f\"Ryukijano/a2c-{env_id}\", # Change the username\n",
        "    commit_message=\"Initial commit\",\n",
        ")"
      ],
      "metadata": {
        "id": "ueuzWVCUTkfS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "a053be61-21d9-43b9-9e98-9cb175b5eeae"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-8e20be2aa313>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m package_to_hub(\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mmodel_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34mf\"a2c-{env_id}\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mmodel_architecture\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"A2C\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0menv_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0menv_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Take a coffee break ☕\n",
        "- You already trained your first robot that learned to move congratutlations 🥳!\n",
        "- It's **time to take a break**. Don't hesitate to **save this notebook** `File > Save a copy to Drive` to work on this second part later.\n"
      ],
      "metadata": {
        "id": "Qk9ykOk9D6Qh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Environment 2: PandaReachDense-v2 🦾\n",
        "\n",
        "The agent we're going to train is a robotic arm that needs to do controls (moving the arm and using the end-effector).\n",
        "\n",
        "In robotics, the *end-effector* is the device at the end of a robotic arm designed to interact with the environment.\n",
        "\n",
        "In `PandaReach`, the robot must place its end-effector at a target position (green ball).\n",
        "\n",
        "We're going to use the dense version of this environment. It means we'll get a *dense reward function* that **will provide a reward at each timestep** (the closer the agent is to completing the task, the higher the reward). Contrary to a *sparse reward function* where the environment **return a reward if and only if the task is completed**.\n",
        "\n",
        "Also, we're going to use the *End-effector displacement control*, it means the **action corresponds to the displacement of the end-effector**. We don't control the individual motion of each joint (joint control).\n",
        "\n",
        "<img src=\"https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/robotics.jpg\"  alt=\"Robotics\"/>\n",
        "\n",
        "\n",
        "This way **the training will be easier**.\n",
        "\n"
      ],
      "metadata": {
        "id": "5VWfwAA7EJg7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "In `PandaReachDense-v2` the robotic arm must place its end-effector at a target position (green ball).\n",
        "\n"
      ],
      "metadata": {
        "id": "oZ7FyDEi7G3T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import gym\n",
        "\n",
        "env_id = \"PandaReachDense-v2\"\n",
        "\n",
        "# Create the env\n",
        "env = gym.make(env_id)\n",
        "\n",
        "# Get the state space and action space\n",
        "s_size = env.observation_space.shape\n",
        "a_size = env.action_space"
      ],
      "metadata": {
        "id": "zXzAu3HYF1WD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"_____OBSERVATION SPACE_____ \\n\")\n",
        "print(\"The State Space is: \", s_size)\n",
        "print(\"Sample observation\", env.observation_space.sample()) # Get a random observation"
      ],
      "metadata": {
        "id": "E-U9dexcF-FB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "67840fc4-6d5d-4274-b72d-8ec3c2cd0cb9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "_____OBSERVATION SPACE_____ \n",
            "\n",
            "The State Space is:  None\n",
            "Sample observation OrderedDict([('achieved_goal', array([ 3.781021 , -1.4800726, -2.3923664], dtype=float32)), ('desired_goal', array([ 8.7986965, -9.593144 , -8.631029 ], dtype=float32)), ('observation', array([-5.7332025,  4.7318764,  4.1732335, -7.448975 , -0.9054228,\n",
            "        8.517136 ], dtype=float32))])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The observation space **is a dictionary with 3 different elements**:\n",
        "- `achieved_goal`: (x,y,z) position of the goal.\n",
        "- `desired_goal`: (x,y,z) distance between the goal position and the current object position.\n",
        "- `observation`: position (x,y,z) and velocity of the end-effector (vx, vy, vz).\n",
        "\n",
        "Given it's a dictionary as observation, **we will need to use a MultiInputPolicy policy instead of MlpPolicy**."
      ],
      "metadata": {
        "id": "g_JClfElGFnF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"\\n _____ACTION SPACE_____ \\n\")\n",
        "print(\"The Action Space is: \", a_size)\n",
        "print(\"Action Space Sample\", env.action_space.sample()) # Take a random action"
      ],
      "metadata": {
        "id": "ib1Kxy4AF-FC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "64484c7a-e965-471e-baf1-a4c9830e9fba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " _____ACTION SPACE_____ \n",
            "\n",
            "The Action Space is:  Box([-1. -1. -1.], [1. 1. 1.], (3,), float32)\n",
            "Action Space Sample [-0.5504403  -0.7761145  -0.08316634]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The action space is a vector with 3 values:\n",
        "- Control x, y, z movement"
      ],
      "metadata": {
        "id": "5MHTHEHZS4yp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now it's your turn:\n",
        "\n",
        "1. Define the environment called \"PandaReachDense-v2\"\n",
        "2. Make a vectorized environment\n",
        "3. Add a wrapper to normalize the observations and rewards. [Check the documentation](https://stable-baselines3.readthedocs.io/en/master/guide/vec_envs.html#vecnormalize)\n",
        "4. Create the A2C Model (don't forget verbose=1 to print the training logs).\n",
        "5. Train it for 1M Timesteps\n",
        "6. Save the model and  VecNormalize statistics when saving the agent\n",
        "7. Evaluate your agent\n",
        "8. Publish your trained model on the Hub 🔥 with `package_to_hub`"
      ],
      "metadata": {
        "id": "nIhPoc5t9HjG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Solution (fill the todo)"
      ],
      "metadata": {
        "id": "sKGbFXZq9ikN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 1 - 2\n",
        "env_id = \"PandaReachDense-v2\"\n",
        "env = make_vec_env(env_id, n_envs=100)\n",
        "\n",
        "# 3\n",
        "env = VecNormalize(env, norm_obs=True, norm_reward=False, clip_obs=10.)\n",
        "\n",
        "# 4\n",
        "model = A2C(policy = \"MultiInputPolicy\",\n",
        "            env = env,\n",
        "            device = \"cuda\",\n",
        "            verbose=1)\n",
        "# 5\n",
        "model.learn(1_000_000)"
      ],
      "metadata": {
        "id": "J-cC-Feg9iMm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "77f21295-54bb-4c69-fc4b-2fa250f37ee9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cuda device\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -9.47    |\n",
            "| time/                 |          |\n",
            "|    fps                | 408      |\n",
            "|    iterations         | 100      |\n",
            "|    time_elapsed       | 122      |\n",
            "|    total_timesteps    | 50000    |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.22    |\n",
            "|    explained_variance | 0.943    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 99       |\n",
            "|    policy_loss        | -2.16    |\n",
            "|    std                | 0.988    |\n",
            "|    value_loss         | 0.587    |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -8.09    |\n",
            "| time/                 |          |\n",
            "|    fps                | 409      |\n",
            "|    iterations         | 200      |\n",
            "|    time_elapsed       | 244      |\n",
            "|    total_timesteps    | 100000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.23    |\n",
            "|    explained_variance | 0.917    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 199      |\n",
            "|    policy_loss        | -0.54    |\n",
            "|    std                | 0.993    |\n",
            "|    value_loss         | 0.163    |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -6.2     |\n",
            "| time/                 |          |\n",
            "|    fps                | 414      |\n",
            "|    iterations         | 300      |\n",
            "|    time_elapsed       | 361      |\n",
            "|    total_timesteps    | 150000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.23    |\n",
            "|    explained_variance | 0.976    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 299      |\n",
            "|    policy_loss        | 0.061    |\n",
            "|    std                | 0.991    |\n",
            "|    value_loss         | 0.0599   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -4.45    |\n",
            "| time/                 |          |\n",
            "|    fps                | 419      |\n",
            "|    iterations         | 400      |\n",
            "|    time_elapsed       | 477      |\n",
            "|    total_timesteps    | 200000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.17    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 399      |\n",
            "|    policy_loss        | 0.116    |\n",
            "|    std                | 0.972    |\n",
            "|    value_loss         | 0.0503   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -3.04    |\n",
            "| time/                 |          |\n",
            "|    fps                | 425      |\n",
            "|    iterations         | 500      |\n",
            "|    time_elapsed       | 588      |\n",
            "|    total_timesteps    | 250000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4.1     |\n",
            "|    explained_variance | 0.95     |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 499      |\n",
            "|    policy_loss        | 0.0902   |\n",
            "|    std                | 0.948    |\n",
            "|    value_loss         | 0.0207   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -2.49    |\n",
            "| time/                 |          |\n",
            "|    fps                | 431      |\n",
            "|    iterations         | 600      |\n",
            "|    time_elapsed       | 695      |\n",
            "|    total_timesteps    | 300000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -4       |\n",
            "|    explained_variance | 0.91     |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 599      |\n",
            "|    policy_loss        | 0.029    |\n",
            "|    std                | 0.919    |\n",
            "|    value_loss         | 0.0156   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -2.09    |\n",
            "| time/                 |          |\n",
            "|    fps                | 435      |\n",
            "|    iterations         | 700      |\n",
            "|    time_elapsed       | 803      |\n",
            "|    total_timesteps    | 350000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.91    |\n",
            "|    explained_variance | 0.958    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 699      |\n",
            "|    policy_loss        | 0.0274   |\n",
            "|    std                | 0.891    |\n",
            "|    value_loss         | 0.00548  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.87    |\n",
            "| time/                 |          |\n",
            "|    fps                | 440      |\n",
            "|    iterations         | 800      |\n",
            "|    time_elapsed       | 907      |\n",
            "|    total_timesteps    | 400000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.8     |\n",
            "|    explained_variance | 0.942    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 799      |\n",
            "|    policy_loss        | -0.0242  |\n",
            "|    std                | 0.857    |\n",
            "|    value_loss         | 0.00539  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.8     |\n",
            "| time/                 |          |\n",
            "|    fps                | 443      |\n",
            "|    iterations         | 900      |\n",
            "|    time_elapsed       | 1015     |\n",
            "|    total_timesteps    | 450000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.69    |\n",
            "|    explained_variance | 0.937    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 899      |\n",
            "|    policy_loss        | -0.0213  |\n",
            "|    std                | 0.827    |\n",
            "|    value_loss         | 0.00418  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.74    |\n",
            "| time/                 |          |\n",
            "|    fps                | 446      |\n",
            "|    iterations         | 1000     |\n",
            "|    time_elapsed       | 1120     |\n",
            "|    total_timesteps    | 500000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.59    |\n",
            "|    explained_variance | 0.948    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 999      |\n",
            "|    policy_loss        | -0.0258  |\n",
            "|    std                | 0.8      |\n",
            "|    value_loss         | 0.00316  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.67    |\n",
            "| time/                 |          |\n",
            "|    fps                | 448      |\n",
            "|    iterations         | 1100     |\n",
            "|    time_elapsed       | 1225     |\n",
            "|    total_timesteps    | 550000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.49    |\n",
            "|    explained_variance | 0.955    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1099     |\n",
            "|    policy_loss        | -0.0069  |\n",
            "|    std                | 0.774    |\n",
            "|    value_loss         | 0.00245  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.63    |\n",
            "| time/                 |          |\n",
            "|    fps                | 450      |\n",
            "|    iterations         | 1200     |\n",
            "|    time_elapsed       | 1332     |\n",
            "|    total_timesteps    | 600000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.38    |\n",
            "|    explained_variance | 0.927    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1199     |\n",
            "|    policy_loss        | -0.0183  |\n",
            "|    std                | 0.747    |\n",
            "|    value_loss         | 0.00739  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.62    |\n",
            "| time/                 |          |\n",
            "|    fps                | 452      |\n",
            "|    iterations         | 1300     |\n",
            "|    time_elapsed       | 1436     |\n",
            "|    total_timesteps    | 650000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.28    |\n",
            "|    explained_variance | 0.969    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1299     |\n",
            "|    policy_loss        | -0.0225  |\n",
            "|    std                | 0.721    |\n",
            "|    value_loss         | 0.00176  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.61    |\n",
            "| time/                 |          |\n",
            "|    fps                | 453      |\n",
            "|    iterations         | 1400     |\n",
            "|    time_elapsed       | 1541     |\n",
            "|    total_timesteps    | 700000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.19    |\n",
            "|    explained_variance | 0.983    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1399     |\n",
            "|    policy_loss        | -0.0269  |\n",
            "|    std                | 0.7      |\n",
            "|    value_loss         | 0.0021   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.58    |\n",
            "| time/                 |          |\n",
            "|    fps                | 456      |\n",
            "|    iterations         | 1500     |\n",
            "|    time_elapsed       | 1641     |\n",
            "|    total_timesteps    | 750000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3.11    |\n",
            "|    explained_variance | 0.972    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1499     |\n",
            "|    policy_loss        | 0.00552  |\n",
            "|    std                | 0.682    |\n",
            "|    value_loss         | 0.00175  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.6     |\n",
            "| time/                 |          |\n",
            "|    fps                | 458      |\n",
            "|    iterations         | 1600     |\n",
            "|    time_elapsed       | 1745     |\n",
            "|    total_timesteps    | 800000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -3       |\n",
            "|    explained_variance | 0.885    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1599     |\n",
            "|    policy_loss        | 0.0169   |\n",
            "|    std                | 0.658    |\n",
            "|    value_loss         | 0.0107   |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.55    |\n",
            "| time/                 |          |\n",
            "|    fps                | 460      |\n",
            "|    iterations         | 1700     |\n",
            "|    time_elapsed       | 1846     |\n",
            "|    total_timesteps    | 850000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -2.92    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1699     |\n",
            "|    policy_loss        | -0.00887 |\n",
            "|    std                | 0.64     |\n",
            "|    value_loss         | 0.00117  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.58    |\n",
            "| time/                 |          |\n",
            "|    fps                | 461      |\n",
            "|    iterations         | 1800     |\n",
            "|    time_elapsed       | 1948     |\n",
            "|    total_timesteps    | 900000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -2.82    |\n",
            "|    explained_variance | 0.959    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1799     |\n",
            "|    policy_loss        | 0.00889  |\n",
            "|    std                | 0.619    |\n",
            "|    value_loss         | 0.00479  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.57    |\n",
            "| time/                 |          |\n",
            "|    fps                | 463      |\n",
            "|    iterations         | 1900     |\n",
            "|    time_elapsed       | 2051     |\n",
            "|    total_timesteps    | 950000   |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -2.7     |\n",
            "|    explained_variance | 0.943    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1899     |\n",
            "|    policy_loss        | -0.0162  |\n",
            "|    std                | 0.594    |\n",
            "|    value_loss         | 0.00262  |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| rollout/              |          |\n",
            "|    ep_len_mean        | 50       |\n",
            "|    ep_rew_mean        | -1.5     |\n",
            "| time/                 |          |\n",
            "|    fps                | 463      |\n",
            "|    iterations         | 2000     |\n",
            "|    time_elapsed       | 2156     |\n",
            "|    total_timesteps    | 1000000  |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -2.59    |\n",
            "|    explained_variance | 0.985    |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1999     |\n",
            "|    policy_loss        | -0.0135  |\n",
            "|    std                | 0.574    |\n",
            "|    value_loss         | 0.00136  |\n",
            "------------------------------------\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.a2c.a2c.A2C at 0x7f8e59d30c40>"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 6\n",
        "model_name = \"a2c-PandaReachDense-v2\"; \n",
        "model.save(model_name)\n",
        "env.save(\"vec_normalize.pkl\")\n",
        "\n",
        "# 7\n",
        "from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize\n",
        "\n",
        "# Load the saved statistics\n",
        "eval_env = DummyVecEnv([lambda: gym.make(\"PandaReachDense-v2\")])\n",
        "eval_env = VecNormalize.load(\"vec_normalize.pkl\", eval_env)\n",
        "\n",
        "#  do not update them at test time\n",
        "eval_env.training = False\n",
        "# reward normalization is not needed at test time\n",
        "eval_env.norm_reward = False\n",
        "\n",
        "# Load the agent\n",
        "model = A2C.load(model_name)\n",
        "\n",
        "mean_reward, std_reward = evaluate_policy(model, eval_env)\n",
        "\n",
        "print(f\"Mean reward = {mean_reward:.2f} +/- {std_reward:.2f}\")\n",
        "\n",
        "# 8\n",
        "package_to_hub(\n",
        "    model=model,\n",
        "    model_name=f\"a2c-{env_id}\",\n",
        "    model_architecture=\"A2C\",\n",
        "    env_id=env_id,\n",
        "    eval_env=eval_env,\n",
        "    repo_id=f\"Ryukijano/a2c-{env_id}\", # TODO: Change the username\n",
        "    commit_message=\"Initial commit\",\n",
        ")"
      ],
      "metadata": {
        "id": "-UnlKLmpg80p",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 806,
          "referenced_widgets": [
            "82e17bdfe46c418f94ba6d0fd8d5db73",
            "8f6751d08c17414f96b2194d770adf09",
            "a004d15844f049aeb404143c9080799a",
            "aa6969be7ead4649ad1a6da583b3f7c2",
            "c67a09a38db24eadb429806bf8000ac1",
            "21051fb7bf9f4dc1ae5916132b2f1fa9",
            "c26d255828ff4d479aa8723000090a10",
            "2756dbef561440339806b0207906571d",
            "ebe2811bc4564eccbccfbc8293ae510b",
            "67df91f24db846d991b086346ae82956",
            "46428c01fdbb4640941b79aed148ee7d",
            "e7622a22b2d1466e97a96f1530c76af0",
            "8801f1dac6b04097b4e303cb84329807",
            "cebaf557c2eb4fdb83bd76951f1c93ef",
            "e9661091ad5447bc850bd1530705b7a7",
            "9ff1b86161504a32871300c403c19c65",
            "bc926b5a4cb34df0a21b49e20f298a1c",
            "c8eec08153bd40a08a8ad0b43af3ea10",
            "796766469caf44f0bcc60049fcb17729",
            "ebfc65884bf941758fd533ddb2dfa3fa",
            "640a141d4e664ea7b3d446f01118e197",
            "69aa3a08e8a44509b895861f2c11eaed",
            "7f5626fff7bf47aebdccc146eb51a53d",
            "4241d484a1c745399b041a1f01f00e85",
            "b7cefd736ac0410398a2101ab623ace3",
            "4851f5860f3840a184efbf5191b7b718",
            "475f3c596e7f417488c66b1c857ce988",
            "afcab5cc1221428baeeb6f6dbb6957c2",
            "0b21d71be2b24e7c928749f692193b61",
            "3db8dadc5c9a4c5e97b1357ab2d599e5",
            "e129a7b74fab41a28184c0d73b9f564f",
            "da94a1dd564f493f8a1edcc77085ccda",
            "dbbc8e5fa5e748d2906ef046e1ecfcbb",
            "602e35a5e3ec41328f398f1c1ab2fbfb",
            "26ce40a4b2404720b0c5d09df046f4d5",
            "e3668527d6d94c2c83593e9cf76ffb10",
            "bc441b469d4a4af5a1c687c4f620be6e",
            "2e90b814d11d4e61aa1c94084bb4da93",
            "44d573ff033f4bbda0254d8b86b7b498",
            "f7f07ad990084942b0096ada6dce88a0",
            "87e8824d2e974696bda9ba37255eaf7b",
            "d822c1ad64554a1282c6025d7b2755b5",
            "a09d67e563a84a86a494c41b7ad28640",
            "739a9f1e952a4a8097e6176c9dc7354d",
            "f69a1e606634451f9a6499988bd6dca1",
            "828ec510aa114d2b8c281e609aba3c9e",
            "9d47051358ee4d908d49af3a0a438005",
            "100899cc1e7546a9ab1d6018203ef92c",
            "4867a4e62ad44be39c47a90838d7ea72",
            "c44f06ce8c1d44e59170acc57dcbf799",
            "c5d0d0af031448a481c185443642d87d",
            "bb1d52a93e454686a2c19d115a3f135d",
            "e578ec5e4d9844d1a3fbf3aecdea0c70",
            "5a1c9dfd0f714ab795debb0701d095dc",
            "e20f44e3bfc54be0be9070d4d1582f2c"
          ]
        },
        "outputId": "734e0b8c-790a-44c2-8371-daa02b549b3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mean reward = -1.42 +/- 0.18\n",
            "\u001b[38;5;4mℹ This function will save, evaluate, generate a video of your agent,\n",
            "create a model card and push everything to the hub. It might take up to 1min.\n",
            "This is a work in progress: if you encounter a bug, please open an issue.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/panda_gym/pybullet.py:97: UserWarning: The use of the render method is not recommended when the environment has not been created with render=True. The rendering will probably be weird. Prefer making the environment with option `render=True`. For example: `env = gym.make('PandaReach-v2', render=True)`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/panda_gym/pybullet.py:97: UserWarning: The use of the render method is not recommended when the environment has not been created with render=True. The rendering will probably be weird. Prefer making the environment with option `render=True`. For example: `env = gym.make('PandaReach-v2', render=True)`.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving video to /tmp/tmpq911h3al/-step-0-to-step-1000.mp4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <function VecVideoRecorder.__del__ at 0x7f8e608dfd30>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/stable_baselines3/common/vec_env/vec_video_recorder.py\", line 113, in __del__\n",
            "    self.close()\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/stable_baselines3/common/vec_env/vec_video_recorder.py\", line 109, in close\n",
            "    VecEnvWrapper.close(self)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/stable_baselines3/common/vec_env/base_vec_env.py\", line 279, in close\n",
            "    return self.venv.close()\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/stable_baselines3/common/vec_env/base_vec_env.py\", line 279, in close\n",
            "    return self.venv.close()\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/stable_baselines3/common/vec_env/dummy_vec_env.py\", line 80, in close\n",
            "    env.close()\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/gym/core.py\", line 298, in close\n",
            "    return self.env.close()\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/panda_gym/envs/core.py\", line 279, in close\n",
            "    self.sim.close()\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/panda_gym/pybullet.py\", line 59, in close\n",
            "    self.physics_client.disconnect()\n",
            "pybullet.error: Not connected to physics server.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[38;5;4mℹ Pushing repo Ryukijano/a2c-PandaReachDense-v2 to the Hugging Face\n",
            "Hub\u001b[0m\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "policy.optimizer.pth:   0%|          | 0.00/44.7k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "82e17bdfe46c418f94ba6d0fd8d5db73"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "policy.pth:   0%|          | 0.00/46.0k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e7622a22b2d1466e97a96f1530c76af0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Upload 4 LFS files:   0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7f5626fff7bf47aebdccc146eb51a53d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vec_normalize.pkl:   0%|          | 0.00/3.06k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "602e35a5e3ec41328f398f1c1ab2fbfb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "a2c-PandaReachDense-v2.zip:   0%|          | 0.00/153k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f69a1e606634451f9a6499988bd6dca1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[38;5;4mℹ Your model is pushed to the Hub. You can view your model here:\n",
            "https://huggingface.co/Ryukijano/a2c-PandaReachDense-v2/tree/main/\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'https://huggingface.co/Ryukijano/a2c-PandaReachDense-v2/tree/main/'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Some additional challenges 🏆\n",
        "The best way to learn **is to try things by your own**! Why not trying  `HalfCheetahBulletEnv-v0` for PyBullet and `PandaPickAndPlace-v1` for Panda-Gym?\n",
        "\n",
        "If you want to try more advanced tasks for panda-gym, you need to check what was done using **TQC or SAC** (a more sample-efficient algorithm suited for robotics tasks). In real robotics, you'll use a more sample-efficient algorithm for a simple reason: contrary to a simulation **if you move your robotic arm too much, you have a risk of breaking it**.\n",
        "\n",
        "PandaPickAndPlace-v1: https://huggingface.co/sb3/tqc-PandaPickAndPlace-v1\n",
        "\n",
        "And don't hesitate to check panda-gym documentation here: https://panda-gym.readthedocs.io/en/latest/usage/train_with_sb3.html\n",
        "\n",
        "Here are some ideas to achieve so:\n",
        "* Train more steps\n",
        "* Try different hyperparameters by looking at what your classmates have done 👉 https://huggingface.co/models?other=https://huggingface.co/models?other=AntBulletEnv-v0\n",
        "* **Push your new trained model** on the Hub 🔥\n"
      ],
      "metadata": {
        "id": "G3xy3Nf3c2O1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "See you on Unit 7! 🔥\n",
        "## Keep learning, stay awesome 🤗"
      ],
      "metadata": {
        "id": "usatLaZ8dM4P"
      }
    }
  ]
}