seminars/sem6/TransferLearningSeminar.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TL_seminar.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "adb6096cdb24400999f97c46b65f51cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1411f3b78cbb4908a8547e4d9a20ba14",
              "IPY_MODEL_46688b329b414aa082baa71663aa3933",
              "IPY_MODEL_d92eae0e0cc54160ab6616028bfef512"
            ],
            "layout": "IPY_MODEL_0196f18fce4f4fe09a9040bebef7a2a6"
          }
        },
        "1411f3b78cbb4908a8547e4d9a20ba14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b61f2e3dfd44e48ab1a1fe54c83905b",
            "placeholder": "​",
            "style": "IPY_MODEL_a2593590e5a042b0b1860c616aaf37de",
            "value": "Downloading: 100%"
          }
        },
        "46688b329b414aa082baa71663aa3933": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6ce2ac5555d4e239f1bbf5feaff3de7",
            "max": 629,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_721af094cfd744c7b7afc48d9639f36f",
            "value": 629
          }
        },
        "d92eae0e0cc54160ab6616028bfef512": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d1044a2cb2f4e8ab515214447e5f977",
            "placeholder": "​",
            "style": "IPY_MODEL_42424d795ad34166b15406baf81b7992",
            "value": " 629/629 [00:00&lt;00:00, 12.6kB/s]"
          }
        },
        "0196f18fce4f4fe09a9040bebef7a2a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b61f2e3dfd44e48ab1a1fe54c83905b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a2593590e5a042b0b1860c616aaf37de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f6ce2ac5555d4e239f1bbf5feaff3de7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "721af094cfd744c7b7afc48d9639f36f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6d1044a2cb2f4e8ab515214447e5f977": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42424d795ad34166b15406baf81b7992": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "18c3bbe6acd2445995db1af07e7afc06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2017f7e425eb42eeb142896c2a767d48",
              "IPY_MODEL_b715db5df81b4bea8ce981a481ea1683",
              "IPY_MODEL_3816223f5fc94b5797fc7c0da8ea77f9"
            ],
            "layout": "IPY_MODEL_b2bac420982e41528863694a37d0ba28"
          }
        },
        "2017f7e425eb42eeb142896c2a767d48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67ac9cc8690c4d619d0e65b08ef01e9c",
            "placeholder": "​",
            "style": "IPY_MODEL_c54992f490f94a88acf9b5df29a301cf",
            "value": "Downloading: 100%"
          }
        },
        "b715db5df81b4bea8ce981a481ea1683": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0edcd75bfbe940b68099a7d12f0b4807",
            "max": 267844284,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6aefbf4b7fed495ca0bcc6c4a3123ef0",
            "value": 267844284
          }
        },
        "3816223f5fc94b5797fc7c0da8ea77f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bdc32575abd644d194d1c47a2fc28e87",
            "placeholder": "​",
            "style": "IPY_MODEL_0d3e0c25dd714025b3c89cd93a4290ba",
            "value": " 255M/255M [00:08&lt;00:00, 37.6MB/s]"
          }
        },
        "b2bac420982e41528863694a37d0ba28": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "67ac9cc8690c4d619d0e65b08ef01e9c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c54992f490f94a88acf9b5df29a301cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0edcd75bfbe940b68099a7d12f0b4807": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6aefbf4b7fed495ca0bcc6c4a3123ef0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bdc32575abd644d194d1c47a2fc28e87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0d3e0c25dd714025b3c89cd93a4290ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c0057f6793bd472f8c680163a9ebbf36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_375fcc9e585647d1bed6119cc61b9c6a",
              "IPY_MODEL_fb673d6dfa754e688cedaa57b662cda3",
              "IPY_MODEL_12797c571e96450085a07bec7c974fd1"
            ],
            "layout": "IPY_MODEL_0ed2f98aa6ac4c3d93814ee2f7f2a63b"
          }
        },
        "375fcc9e585647d1bed6119cc61b9c6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_de0a24eca579442ab7436013676037b2",
            "placeholder": "​",
            "style": "IPY_MODEL_c9f4aceee4ab414187f0ff13187e61b3",
            "value": "Downloading: 100%"
          }
        },
        "fb673d6dfa754e688cedaa57b662cda3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e58032eabfe942e6ba0fdea521e588b1",
            "max": 48,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5b41bdd79d044c0a9003b416e8550909",
            "value": 48
          }
        },
        "12797c571e96450085a07bec7c974fd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ab408fec22240bf9d61353dbf2c0ae9",
            "placeholder": "​",
            "style": "IPY_MODEL_54f237d1f27a4bc69b27ec4759d6e133",
            "value": " 48.0/48.0 [00:00&lt;00:00, 1.27kB/s]"
          }
        },
        "0ed2f98aa6ac4c3d93814ee2f7f2a63b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de0a24eca579442ab7436013676037b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9f4aceee4ab414187f0ff13187e61b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e58032eabfe942e6ba0fdea521e588b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b41bdd79d044c0a9003b416e8550909": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7ab408fec22240bf9d61353dbf2c0ae9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54f237d1f27a4bc69b27ec4759d6e133": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3131868617924113a5bab43a04c51b43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_92943f10573e4e7dbfce95727a9d5a8e",
              "IPY_MODEL_eb18f8991a0c4960b9f2e12b4b5161b2",
              "IPY_MODEL_030e2687c7a94bcc8be2212d6d2b422d"
            ],
            "layout": "IPY_MODEL_50436cffddb54231892d489e75e0b0c3"
          }
        },
        "92943f10573e4e7dbfce95727a9d5a8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78c791ef97b047d1990f7f5e2e20e08e",
            "placeholder": "​",
            "style": "IPY_MODEL_2e0b96fcdf60463b9ce12fbfa49f5a77",
            "value": "Downloading: 100%"
          }
        },
        "eb18f8991a0c4960b9f2e12b4b5161b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cee8efa29723486189042ba98a7f5e2a",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ff338f71f21e4c78a2d75e00807dd82d",
            "value": 231508
          }
        },
        "030e2687c7a94bcc8be2212d6d2b422d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_649f07edc7104070823342c4d1bd938b",
            "placeholder": "​",
            "style": "IPY_MODEL_02036186913c4c8eab1ad90b707b11b8",
            "value": " 226k/226k [00:00&lt;00:00, 972kB/s]"
          }
        },
        "50436cffddb54231892d489e75e0b0c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78c791ef97b047d1990f7f5e2e20e08e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e0b96fcdf60463b9ce12fbfa49f5a77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cee8efa29723486189042ba98a7f5e2a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ff338f71f21e4c78a2d75e00807dd82d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "649f07edc7104070823342c4d1bd938b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02036186913c4c8eab1ad90b707b11b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "74b3acedd9bd47a08ae4814a287bd036": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3063cde4f74f484f9a85b9334fbfc746",
              "IPY_MODEL_c95c5b54f4384b0d9d5fc20f22a26c43",
              "IPY_MODEL_0bfb809d463d4aed96564ef171732669"
            ],
            "layout": "IPY_MODEL_e26dddfeeb8e47cfa2da4dca9bf13bcc"
          }
        },
        "3063cde4f74f484f9a85b9334fbfc746": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_789aa4f00b2c4118bb15d71ddab0651e",
            "placeholder": "​",
            "style": "IPY_MODEL_930199747902479c8dc07a475c278297",
            "value": "Downloading: 100%"
          }
        },
        "c95c5b54f4384b0d9d5fc20f22a26c43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2f478c8aadb94122bb16a40dbefe070c",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b56e1fe184394cacbe8d1ab99cfeaf20",
            "value": 231508
          }
        },
        "0bfb809d463d4aed96564ef171732669": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a04aba8748204cc28b4c9a7c6a42c5c5",
            "placeholder": "​",
            "style": "IPY_MODEL_d2698d22c3f64ad3abf82f3262312287",
            "value": " 226k/226k [00:00&lt;00:00, 1.13MB/s]"
          }
        },
        "e26dddfeeb8e47cfa2da4dca9bf13bcc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "789aa4f00b2c4118bb15d71ddab0651e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "930199747902479c8dc07a475c278297": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2f478c8aadb94122bb16a40dbefe070c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b56e1fe184394cacbe8d1ab99cfeaf20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a04aba8748204cc28b4c9a7c6a42c5c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d2698d22c3f64ad3abf82f3262312287": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3affc4a483674ae7ba73565569642916": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e4334654445e4b14a28ca5d0d4723ca0",
              "IPY_MODEL_a79baf16f4eb4140a805482c015a3d63",
              "IPY_MODEL_794eecfb24bd40faba43298754623cc4"
            ],
            "layout": "IPY_MODEL_5d993eb3612248bbba59e78985eb5e73"
          }
        },
        "e4334654445e4b14a28ca5d0d4723ca0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b94c29bcdb014ed0b9dc0bc22b44ac5b",
            "placeholder": "​",
            "style": "IPY_MODEL_4fcc7f1d2faf46c9a9087162e79928c3",
            "value": "Downloading: 100%"
          }
        },
        "a79baf16f4eb4140a805482c015a3d63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d4c4edfee49943a782da74a5ae4ed452",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b89f157b283e4a8391077671681ea9a5",
            "value": 28
          }
        },
        "794eecfb24bd40faba43298754623cc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_883ea77aea424801af31533d14deb6f1",
            "placeholder": "​",
            "style": "IPY_MODEL_f693d749a5c14aff999d240e1276757e",
            "value": " 28.0/28.0 [00:00&lt;00:00, 731B/s]"
          }
        },
        "5d993eb3612248bbba59e78985eb5e73": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b94c29bcdb014ed0b9dc0bc22b44ac5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4fcc7f1d2faf46c9a9087162e79928c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d4c4edfee49943a782da74a5ae4ed452": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b89f157b283e4a8391077671681ea9a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "883ea77aea424801af31533d14deb6f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f693d749a5c14aff999d240e1276757e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ff1c6188a1264f52aadba56060eadfeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6f45cea5ab884087a3ea14facdb77021",
              "IPY_MODEL_486a12a3463e49b1ac9d366e9a6a227b",
              "IPY_MODEL_75629a97a0e1477ab000b4fd391d0a89"
            ],
            "layout": "IPY_MODEL_f03e1493b9cc4e86b8f498ba73e4fcf3"
          }
        },
        "6f45cea5ab884087a3ea14facdb77021": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_98a07e68bb204adeacc53615717bafe9",
            "placeholder": "​",
            "style": "IPY_MODEL_8d8f8c73e04640f09c4f05b4195e9097",
            "value": "Downloading: 100%"
          }
        },
        "486a12a3463e49b1ac9d366e9a6a227b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d47236c5db064c9e92c1f46dbd891ff8",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_940c8719a77540e7a8c0e2abb7fd98e3",
            "value": 570
          }
        },
        "75629a97a0e1477ab000b4fd391d0a89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ff7f563728645a8b3e968e23a7db37c",
            "placeholder": "​",
            "style": "IPY_MODEL_254c125aa12f4ca88a79896c1b924fb8",
            "value": " 570/570 [00:00&lt;00:00, 14.3kB/s]"
          }
        },
        "f03e1493b9cc4e86b8f498ba73e4fcf3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "98a07e68bb204adeacc53615717bafe9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d8f8c73e04640f09c4f05b4195e9097": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d47236c5db064c9e92c1f46dbd891ff8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "940c8719a77540e7a8c0e2abb7fd98e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7ff7f563728645a8b3e968e23a7db37c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "254c125aa12f4ca88a79896c1b924fb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4d3b7b3c48d49be9ceaa29a5eb142dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c74bc5a1572d4615966454fed0da693f",
              "IPY_MODEL_9e5dea065c5c423ebcea87bdc8c12a40",
              "IPY_MODEL_add708258e524660b38a5035b96a21c2"
            ],
            "layout": "IPY_MODEL_badca614c7624bbca8558d08ed36bb6b"
          }
        },
        "c74bc5a1572d4615966454fed0da693f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f8de5a819c664db5bf9647a74b41ac29",
            "placeholder": "​",
            "style": "IPY_MODEL_7445076e4a7c403fb6b9b51b3d3ed9fb",
            "value": "Downloading: 100%"
          }
        },
        "9e5dea065c5c423ebcea87bdc8c12a40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2d372770e1a4ddd96ea6d2bd08dda3c",
            "max": 440473133,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_946ce78aa5e7419eaa82f49de837e149",
            "value": 440473133
          }
        },
        "add708258e524660b38a5035b96a21c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_03f122dcdc184aa5b5ed60fc2f010739",
            "placeholder": "​",
            "style": "IPY_MODEL_3c92d3d72b50450a8d29c01d8cf48f12",
            "value": " 420M/420M [00:26&lt;00:00, 11.3MB/s]"
          }
        },
        "badca614c7624bbca8558d08ed36bb6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f8de5a819c664db5bf9647a74b41ac29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7445076e4a7c403fb6b9b51b3d3ed9fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c2d372770e1a4ddd96ea6d2bd08dda3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "946ce78aa5e7419eaa82f49de837e149": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "03f122dcdc184aa5b5ed60fc2f010739": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c92d3d72b50450a8d29c01d8cf48f12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "88d325627e0141a1a99d4e5c38538c68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f23b42d37d414b59b800be306d335df7",
              "IPY_MODEL_e96781ca5a534551bb6025327d54fb31",
              "IPY_MODEL_0be1287add7040c898a4005de7778945"
            ],
            "layout": "IPY_MODEL_ed0f8626f5ec41c7ba4192a043b820c3"
          }
        },
        "f23b42d37d414b59b800be306d335df7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_713e496c4bc24a73b0c6bb1291412f39",
            "placeholder": "​",
            "style": "IPY_MODEL_5cda9cab944d4ee69a0b378b1734d6be",
            "value": "Downloading: 100%"
          }
        },
        "e96781ca5a534551bb6025327d54fb31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e01d67e1125c40b19dc6f2a912b5a0cf",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_563decc39f364d239bb9af680ecee5d7",
            "value": 466062
          }
        },
        "0be1287add7040c898a4005de7778945": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd1db82290f549f19880124ee8e2b7cd",
            "placeholder": "​",
            "style": "IPY_MODEL_bb214d9c3df04182a21f1edd293ac4a2",
            "value": " 455k/455k [00:00&lt;00:00, 1.10MB/s]"
          }
        },
        "ed0f8626f5ec41c7ba4192a043b820c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "713e496c4bc24a73b0c6bb1291412f39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5cda9cab944d4ee69a0b378b1734d6be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e01d67e1125c40b19dc6f2a912b5a0cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "563decc39f364d239bb9af680ecee5d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fd1db82290f549f19880124ee8e2b7cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb214d9c3df04182a21f1edd293ac4a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e00059f36bc145d3a05b59d521c76daa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_95ae4bcb7dbf45108671d4dbdf8cc386",
              "IPY_MODEL_a4a084e20a8045db8f3879f6edc22d52",
              "IPY_MODEL_07a3fdc2c5a049aaabc710e28091d9fa"
            ],
            "layout": "IPY_MODEL_e884f6203b8f4f6abba02bec8cd9af95"
          }
        },
        "95ae4bcb7dbf45108671d4dbdf8cc386": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_98929f9f516f44f6a16795ea55e1caf0",
            "placeholder": "​",
            "style": "IPY_MODEL_8cd43912df8b4d5cad13d02897487432",
            "value": "Downloading: 100%"
          }
        },
        "a4a084e20a8045db8f3879f6edc22d52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a459cd9d0b34884b90d7e027dbf400e",
            "max": 1713123,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_71613e16288a480194b86d2e9ecc71bc",
            "value": 1713123
          }
        },
        "07a3fdc2c5a049aaabc710e28091d9fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2aa303232ae147db8425a2bf9bb659b2",
            "placeholder": "​",
            "style": "IPY_MODEL_3fe62ae12da4439a88c042e22d69b103",
            "value": " 1.63M/1.63M [00:00&lt;00:00, 1.59MB/s]"
          }
        },
        "e884f6203b8f4f6abba02bec8cd9af95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "98929f9f516f44f6a16795ea55e1caf0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8cd43912df8b4d5cad13d02897487432": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6a459cd9d0b34884b90d7e027dbf400e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71613e16288a480194b86d2e9ecc71bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2aa303232ae147db8425a2bf9bb659b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3fe62ae12da4439a88c042e22d69b103": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0808d90bc4a243bb80f36ea3f7d7b062": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c1df4250aeb04179a83287bd420dee7d",
              "IPY_MODEL_f9a2cb6c139c40f992996f7d2aac9880",
              "IPY_MODEL_b86386b7cbd44c31bee16b81efef0531"
            ],
            "layout": "IPY_MODEL_fd38ef412721463cbdabba5cebafa304"
          }
        },
        "c1df4250aeb04179a83287bd420dee7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1adbefab111244d8b198b4f062dc1ccb",
            "placeholder": "​",
            "style": "IPY_MODEL_618aebc4a96e4ecd825d209aea007024",
            "value": "Downloading: 100%"
          }
        },
        "f9a2cb6c139c40f992996f7d2aac9880": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_14aae45587144a19933d9c2dab2a2f42",
            "max": 1270925,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8daf1a40a271418bb12ac88de1890159",
            "value": 1270925
          }
        },
        "b86386b7cbd44c31bee16b81efef0531": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0cc2b7655ca74a6d9f36a679d33c562b",
            "placeholder": "​",
            "style": "IPY_MODEL_25ce556509dd48949920ced5491cdec0",
            "value": " 1.21M/1.21M [00:00&lt;00:00, 4.65MB/s]"
          }
        },
        "fd38ef412721463cbdabba5cebafa304": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1adbefab111244d8b198b4f062dc1ccb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "618aebc4a96e4ecd825d209aea007024": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14aae45587144a19933d9c2dab2a2f42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8daf1a40a271418bb12ac88de1890159": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0cc2b7655ca74a6d9f36a679d33c562b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25ce556509dd48949920ced5491cdec0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a0ff84a01d0e49a4878ceee491c21b74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_600034afb5dc45dda6a848e53f84c5fe",
              "IPY_MODEL_f010200682c14090a43885f381b59bff",
              "IPY_MODEL_c46452c83ad84ae882fcf286905992c5"
            ],
            "layout": "IPY_MODEL_ebc162eb8af84b42a53cbd4f853507e6"
          }
        },
        "600034afb5dc45dda6a848e53f84c5fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f3ae323fda034811a84ecf485ada86c8",
            "placeholder": "​",
            "style": "IPY_MODEL_7b27ce6d076645c1a638ed90ffb7e589",
            "value": "Downloading: 100%"
          }
        },
        "f010200682c14090a43885f381b59bff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9f723e627a9746eba28fc6e35fa1ed0b",
            "max": 609,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9a3e4f11cdf244a388ed18aff392bccf",
            "value": 609
          }
        },
        "c46452c83ad84ae882fcf286905992c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e280f6adbbf44e2e8ad49fb33a831998",
            "placeholder": "​",
            "style": "IPY_MODEL_413157f320f64f829e14c8224e5a2e88",
            "value": " 609/609 [00:00&lt;00:00, 14.5kB/s]"
          }
        },
        "ebc162eb8af84b42a53cbd4f853507e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3ae323fda034811a84ecf485ada86c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b27ce6d076645c1a638ed90ffb7e589": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9f723e627a9746eba28fc6e35fa1ed0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a3e4f11cdf244a388ed18aff392bccf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e280f6adbbf44e2e8ad49fb33a831998": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "413157f320f64f829e14c8224e5a2e88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b266ea1ffe8f4df09fa167391df40fa8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a32d661df906474a88a97f4f88d51de9",
              "IPY_MODEL_2141112c8ea44b618067066ebdf186ff",
              "IPY_MODEL_80fc8d42c19149959d5d2ad69956929c"
            ],
            "layout": "IPY_MODEL_18ddc2ddde6b49ae8fbee85ac0d7af24"
          }
        },
        "a32d661df906474a88a97f4f88d51de9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d99edbd5725c417386f39d7d83d8bcac",
            "placeholder": "​",
            "style": "IPY_MODEL_ced35c4752874f8e9307828a2454552b",
            "value": "Downloading: 100%"
          }
        },
        "2141112c8ea44b618067066ebdf186ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a9534fb470d47439c9eaeb9b6cffcbb",
            "max": 3141928084,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1c6667c42d9644228a974a013acc39ca",
            "value": 3141928084
          }
        },
        "80fc8d42c19149959d5d2ad69956929c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_88633d1e40ef4b1b87295954e1f74636",
            "placeholder": "​",
            "style": "IPY_MODEL_7e1abd404a074de8afcb1f8d47a789f8",
            "value": " 2.93G/2.93G [03:07&lt;00:00, 16.0MB/s]"
          }
        },
        "18ddc2ddde6b49ae8fbee85ac0d7af24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d99edbd5725c417386f39d7d83d8bcac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ced35c4752874f8e9307828a2454552b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2a9534fb470d47439c9eaeb9b6cffcbb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c6667c42d9644228a974a013acc39ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "88633d1e40ef4b1b87295954e1f74636": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e1abd404a074de8afcb1f8d47a789f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-cKMAYCLvK01"
      },
      "source": [
        "# Work with Transformers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TGkPvbYkvOm1"
      },
      "source": [
        "!pip install transformers\n",
        "# To install from source instead of the last release, comment the command above and uncomment the following one.\n",
        "# !pip install git+https://github.com/huggingface/transformers.git\n",
        "import transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kES3ilwExwOQ"
      },
      "source": [
        "# Install SentencePiece library https://github.com/google/sentencepiece#installation\n",
        "!pip install sentencepiece"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Gzw3q0evYmG"
      },
      "source": [
        "GITHUB https://github.com/huggingface/transformers\n",
        "\n",
        "\n",
        "See examples of how to do comon tasks:\n",
        "https://github.com/huggingface/transformers/tree/master/examples\n",
        "\n",
        "\n",
        "All available Hugging Face models you can find here:\n",
        "https://huggingface.co/models\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CvImJQRY2MBl"
      },
      "source": [
        "The library is build around three types of classes for each model:\n",
        "\n",
        "* ***model classes*** e.g., BertModel which are 20+ PyTorch models (torch.nn.Modules) that work with the pretrained weights provided in the library. In TF2, these are tf.keras.Model.\n",
        "\n",
        "* ***configuration classes*** which store all the parameters required to build a model, e.g., BertConfig. You don’t always need to instantiate these your-self. In particular, if you are using a pretrained model without any modification, creating the model will automatically take care of instantiating the configuration (which is part of the model)\n",
        "\n",
        "* ***tokenizer classes*** which store the vocabulary for each model and provide methods for encoding/decoding strings in a list of token embeddings indices to be fed to a model, e.g., BertTokenizer\n",
        "\n",
        "All these classes can be instantiated from pretrained instances and saved locally using two methods:\n",
        "\n",
        "* *from_pretrained()* let you instantiate a model/configuration/tokenizer from a pretrained version either provided by the library itself (currently 27 models are provided as listed here) or stored locally (or on a server) by the user,\n",
        "\n",
        "* *save_pretrained()* let you save a model/configuration/tokenizer locally so that it can be reloaded using from_pretrained().\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1XERl8Xl5m4B",
        "outputId": "7eb74f81-c81a-4358-dad1-fab1ba282b3f"
      },
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "# Allocate a pipeline for question-answering\n",
        "question_answerer = pipeline('question-answering')\n",
        "question_answerer({\n",
        "     'question': 'What is the name of the repository ?',\n",
        "     'context': 'Pipeline have been included in the huggingface/transformers repository'\n",
        "})"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No model was supplied, defaulted to distilbert-base-cased-distilled-squad (https://huggingface.co/distilbert-base-cased-distilled-squad)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'huggingface/transformers',\n",
              " 'end': 59,\n",
              " 'score': 0.5135961174964905,\n",
              " 'start': 35}"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180,
          "referenced_widgets": [
            "adb6096cdb24400999f97c46b65f51cd",
            "1411f3b78cbb4908a8547e4d9a20ba14",
            "46688b329b414aa082baa71663aa3933",
            "d92eae0e0cc54160ab6616028bfef512",
            "0196f18fce4f4fe09a9040bebef7a2a6",
            "5b61f2e3dfd44e48ab1a1fe54c83905b",
            "a2593590e5a042b0b1860c616aaf37de",
            "f6ce2ac5555d4e239f1bbf5feaff3de7",
            "721af094cfd744c7b7afc48d9639f36f",
            "6d1044a2cb2f4e8ab515214447e5f977",
            "42424d795ad34166b15406baf81b7992",
            "18c3bbe6acd2445995db1af07e7afc06",
            "2017f7e425eb42eeb142896c2a767d48",
            "b715db5df81b4bea8ce981a481ea1683",
            "3816223f5fc94b5797fc7c0da8ea77f9",
            "b2bac420982e41528863694a37d0ba28",
            "67ac9cc8690c4d619d0e65b08ef01e9c",
            "c54992f490f94a88acf9b5df29a301cf",
            "0edcd75bfbe940b68099a7d12f0b4807",
            "6aefbf4b7fed495ca0bcc6c4a3123ef0",
            "bdc32575abd644d194d1c47a2fc28e87",
            "0d3e0c25dd714025b3c89cd93a4290ba",
            "c0057f6793bd472f8c680163a9ebbf36",
            "375fcc9e585647d1bed6119cc61b9c6a",
            "fb673d6dfa754e688cedaa57b662cda3",
            "12797c571e96450085a07bec7c974fd1",
            "0ed2f98aa6ac4c3d93814ee2f7f2a63b",
            "de0a24eca579442ab7436013676037b2",
            "c9f4aceee4ab414187f0ff13187e61b3",
            "e58032eabfe942e6ba0fdea521e588b1",
            "5b41bdd79d044c0a9003b416e8550909",
            "7ab408fec22240bf9d61353dbf2c0ae9",
            "54f237d1f27a4bc69b27ec4759d6e133",
            "3131868617924113a5bab43a04c51b43",
            "92943f10573e4e7dbfce95727a9d5a8e",
            "eb18f8991a0c4960b9f2e12b4b5161b2",
            "030e2687c7a94bcc8be2212d6d2b422d",
            "50436cffddb54231892d489e75e0b0c3",
            "78c791ef97b047d1990f7f5e2e20e08e",
            "2e0b96fcdf60463b9ce12fbfa49f5a77",
            "cee8efa29723486189042ba98a7f5e2a",
            "ff338f71f21e4c78a2d75e00807dd82d",
            "649f07edc7104070823342c4d1bd938b",
            "02036186913c4c8eab1ad90b707b11b8"
          ]
        },
        "id": "Sojn1ST35kU5",
        "outputId": "d53114e5-6734-498b-ee51-2dd2a58ed51b"
      },
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "# Allocate a pipeline for sentiment-analysis\n",
        "classifier = pipeline('sentiment-analysis')\n",
        "classifier('We are very happy to use transformers repository.')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/629 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "adb6096cdb24400999f97c46b65f51cd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/255M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "18c3bbe6acd2445995db1af07e7afc06"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c0057f6793bd472f8c680163a9ebbf36"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3131868617924113a5bab43a04c51b43"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'label': 'POSITIVE', 'score': 0.99924635887146}]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R0ejZHXq7PUe"
      },
      "source": [
        "### Tokenizers. Byte Pair Encoding "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c3ZCfhb87Zle"
      },
      "source": [
        "Neural network models need to process text somehow. First, it is required to **encode** text as numbers with ids, since the tensor operations act on numbers. After, if the output of the network are words, it is required to **decode** the predicted tokens ids back to text.\n",
        "\n",
        "Instead of training a machine to lookup fixed dictionaries, we want to teach machines to recognize and “read” text in such a way that it can learn from this action itself. Machines don’t have human knowledge, they need to be told how to break text into standard units to process it. They do this using a system called **tokenization**, where sequences of text are broken into smaller parts, or “tokens”, and then fed as input into a DL NLP model.\n",
        "\n",
        "There are many problems and practical issues with tokenization:\n",
        "* how large our *vocabulary* of words, `vocab_size`, should be and as a result consider memory limitations (If we operate on a word level, we will have to deal with large vocabulary size. If instead we use character-level models, it would take lots of iterations to process a sequence)\n",
        "* what to do with unique rare words and missing tokens (the problem of vocabulary coverage)\n",
        "* what to do with languages without spaces between words?\n",
        "\n",
        "We want a system that is able to deal with missing tokens without needing an infinite vocabulary. We want a tokenization scheme that deals with an infinite potential vocabulary via a finite list of known words. Also, we don’t want the extra complexity of breaking everything into single characters since character-level tokenization can lose some of the meaning and semantic niceties of the word level.\n",
        "\n",
        "We need a compromise, a hybrid between word-level and character-level tokenization called ***subword tokenization***. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YBmpbhOvrW0H"
      },
      "source": [
        "Three main types of subword tokenizers used in Hugging Face library:\n",
        "* BPE (https://arxiv.org/pdf/1508.07909.pdf)\n",
        "* WordPiece\n",
        "* SentencePiece(https://www.aclweb.org/anthology/D18-2012.pdf) unicode characters are grouped together using either a [unigram language model](https://www.aclweb.org/anthology/P18-1007.pdf)\n",
        "\n",
        "Read about tokenizers you can for example here:\n",
        "* https://blog.floydhub.com/tokenization-nlp/#bpe\n",
        "* https://huggingface.co/transformers/tokenizer_summary.html\n",
        "* https://towardsdatascience.com/comparing-transformer-tokenizers-686307856955"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O-1u59G3zsAa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 130,
          "referenced_widgets": [
            "74b3acedd9bd47a08ae4814a287bd036",
            "3063cde4f74f484f9a85b9334fbfc746",
            "c95c5b54f4384b0d9d5fc20f22a26c43",
            "0bfb809d463d4aed96564ef171732669",
            "e26dddfeeb8e47cfa2da4dca9bf13bcc",
            "789aa4f00b2c4118bb15d71ddab0651e",
            "930199747902479c8dc07a475c278297",
            "2f478c8aadb94122bb16a40dbefe070c",
            "b56e1fe184394cacbe8d1ab99cfeaf20",
            "a04aba8748204cc28b4c9a7c6a42c5c5",
            "d2698d22c3f64ad3abf82f3262312287",
            "3affc4a483674ae7ba73565569642916",
            "e4334654445e4b14a28ca5d0d4723ca0",
            "a79baf16f4eb4140a805482c015a3d63",
            "794eecfb24bd40faba43298754623cc4",
            "5d993eb3612248bbba59e78985eb5e73",
            "b94c29bcdb014ed0b9dc0bc22b44ac5b",
            "4fcc7f1d2faf46c9a9087162e79928c3",
            "d4c4edfee49943a782da74a5ae4ed452",
            "b89f157b283e4a8391077671681ea9a5",
            "883ea77aea424801af31533d14deb6f1",
            "f693d749a5c14aff999d240e1276757e",
            "ff1c6188a1264f52aadba56060eadfeb",
            "6f45cea5ab884087a3ea14facdb77021",
            "486a12a3463e49b1ac9d366e9a6a227b",
            "75629a97a0e1477ab000b4fd391d0a89",
            "f03e1493b9cc4e86b8f498ba73e4fcf3",
            "98a07e68bb204adeacc53615717bafe9",
            "8d8f8c73e04640f09c4f05b4195e9097",
            "d47236c5db064c9e92c1f46dbd891ff8",
            "940c8719a77540e7a8c0e2abb7fd98e3",
            "7ff7f563728645a8b3e968e23a7db37c",
            "254c125aa12f4ca88a79896c1b924fb8"
          ]
        },
        "outputId": "b450443a-ba93-497d-d13b-72dc7789fa39"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "# subwords:'gp', '##u'\n",
        "print(tokenizer.tokenize(\"I have a new GPU!\"))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "74b3acedd9bd47a08ae4814a287bd036"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3affc4a483674ae7ba73565569642916"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ff1c6188a1264f52aadba56060eadfeb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['i', 'have', 'a', 'new', 'gp', '##u', '!']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZYGTTqg6xoEA"
      },
      "source": [
        "The `T5Tokenizer` uses SentencePiece for example, where the\n",
        "`\"▁\"` character was included in the vocabulary. Decoding with SentencePiece is very easy since all tokens can just be\n",
        "concatenated and `\"▁\"` is replaced by a space.\n",
        "\n",
        "All transformers models in the library that use SentencePiece use it in combination with unigram. Examples of models\n",
        "using SentencePiece are [ALBERT](https://huggingface.co/transformers/model_doc/albert.html), [XLNet](https://huggingface.co/transformers/model_doc/xlnet.html), [Marian](https://huggingface.co/transformers/model_doc/marian.html), and [T5](https://huggingface.co/transformers/model_doc/t5.html)."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import T5Tokenizer"
      ],
      "metadata": {
        "id": "WsF8jxmEun_X"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lS8z0xMBxrwn",
        "outputId": "89ac7276-faa2-400b-c455-02c3f1ef6d78"
      },
      "source": [
        "from transformers import T5Tokenizer\n",
        "\n",
        "tokenizer = T5Tokenizer.from_pretrained(\"sberbank-ai/ruT5-large\")\n",
        "print(tokenizer.tokenize(\"Don't you love 🤗 Transformers? We sure do.\"))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['▁D', 'on', \"'\", 't', '▁you', '▁lo', 've', '▁', '🤗', '▁Trans', 'form', 'ers', '?', '▁We', '▁s', 'ure', '▁do', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0VOAlTxTvmn3"
      },
      "source": [
        "### BERT example \n",
        "\n",
        "preparing a tokenized input (a list of token embeddings indices to be fed to Bert) from a text string "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fG6QrwW7vkxb"
      },
      "source": [
        "import torch\n",
        "from transformers import BertTokenizer, BertModel, BertForMaskedLM\n",
        "\n",
        "import logging\n",
        "logging.basicConfig(level=logging.INFO)\n",
        "\n",
        "# Load pre-trained model tokenizer (vocabulary)\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Tokenize input\n",
        "text = \"[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]\"\n",
        "tokenized_text = tokenizer.tokenize(text)\n",
        "\n",
        "# Mask a token that we will try to predict back with `BertForMaskedLM`\n",
        "masked_index = 8\n",
        "tokenized_text[masked_index] = '[MASK]'\n",
        "assert tokenized_text == ['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', '[MASK]', 'was', 'a', 'puppet', '##eer', '[SEP]']\n",
        "\n",
        "# Convert token to vocabulary indices\n",
        "indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "# Define sentence A and B indices associated to 1st and 2nd sentences\n",
        "segments_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]\n",
        "\n",
        "# Convert inputs to PyTorch tensors\n",
        "tokens_tensor = torch.tensor([indexed_tokens])\n",
        "segments_tensors = torch.tensor([segments_ids])"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JfdEZp6rFbA4"
      },
      "source": [
        "Use `BertForMaskedLM` for predicting MASK token."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156,
          "referenced_widgets": [
            "c4d3b7b3c48d49be9ceaa29a5eb142dd",
            "c74bc5a1572d4615966454fed0da693f",
            "9e5dea065c5c423ebcea87bdc8c12a40",
            "add708258e524660b38a5035b96a21c2",
            "badca614c7624bbca8558d08ed36bb6b",
            "f8de5a819c664db5bf9647a74b41ac29",
            "7445076e4a7c403fb6b9b51b3d3ed9fb",
            "c2d372770e1a4ddd96ea6d2bd08dda3c",
            "946ce78aa5e7419eaa82f49de837e149",
            "03f122dcdc184aa5b5ed60fc2f010739",
            "3c92d3d72b50450a8d29c01d8cf48f12"
          ]
        },
        "id": "OHSQs37Yvk6q",
        "outputId": "6f26b309-d8eb-42c0-c5f3-588437589b3f"
      },
      "source": [
        "model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n",
        "model.eval()\n",
        "\n",
        "# If you have a GPU, put everything on cuda\n",
        "tokens_tensor = tokens_tensor.to('cuda')\n",
        "segments_tensors = segments_tensors.to('cuda')\n",
        "model.to('cuda')\n",
        "\n",
        "# Predict all tokens\n",
        "with torch.no_grad():\n",
        "    outputs = model(tokens_tensor, token_type_ids=segments_tensors)\n",
        "    predictions = outputs[0]\n",
        "\n",
        "# confirm we were able to predict 'henson'\n",
        "predicted_index = torch.argmax(predictions[0, masked_index]).item()\n",
        "print(predicted_index)\n",
        "predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]\n",
        "print(predicted_token)\n",
        "assert predicted_token == 'henson'"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/420M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c4d3b7b3c48d49be9ceaa29a5eb142dd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "27227\n",
            "henson\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A5rgAsCUvkzz",
        "outputId": "32e825b0-1bda-485e-ae59-e778a79204a1"
      },
      "source": [
        "# Load pre-trained model (weights)\n",
        "model = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Set the model in evaluation mode to deactivate the DropOut modules\n",
        "# This is IMPORTANT to have reproducible results during evaluation!\n",
        "model.eval()\n",
        "\n",
        "# If you have a GPU, put everything on cuda\n",
        "tokens_tensor = tokens_tensor.to('cuda')\n",
        "segments_tensors = segments_tensors.to('cuda')\n",
        "model.to('cuda')\n",
        "\n",
        "# Predict hidden states features for each layer\n",
        "with torch.no_grad():\n",
        "    outputs = model(tokens_tensor, token_type_ids=segments_tensors)\n",
        "    # Transformers models always output tuples.\n",
        "    # In our case, the first element is the hidden state of the last layer of the Bert model\n",
        "    encoded_layers = outputs[0]\n",
        "\n",
        "print(encoded_layers)\n",
        "# We have encoded our input sequence in a FloatTensor of shape (batch size, sequence length, model hidden dimension)\n",
        "assert tuple(encoded_layers.shape) == (1, len(indexed_tokens), model.config.hidden_size)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[[-0.5570,  0.2839, -0.6436,  ..., -0.7274,  0.4557,  0.6204],\n",
            "         [-1.1572,  0.0354,  0.0355,  ...,  0.0591,  0.1097, -0.3150],\n",
            "         [ 0.1008, -0.5286, -0.4688,  ..., -0.0431,  0.4889,  0.4134],\n",
            "         ...,\n",
            "         [ 0.1948,  0.0761,  0.2893,  ..., -0.0807,  0.7071,  0.0502],\n",
            "         [-0.1119,  0.0714,  0.6101,  ...,  0.4044,  0.1614, -0.3569],\n",
            "         [ 0.8296,  0.2729, -0.3090,  ...,  0.2452, -0.3581, -0.1970]]],\n",
            "       device='cuda:0')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U-63iBdO6Mf3"
      },
      "source": [
        "##### Check sentence embeddings of the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451,
          "referenced_widgets": [
            "88d325627e0141a1a99d4e5c38538c68",
            "f23b42d37d414b59b800be306d335df7",
            "e96781ca5a534551bb6025327d54fb31",
            "0be1287add7040c898a4005de7778945",
            "ed0f8626f5ec41c7ba4192a043b820c3",
            "713e496c4bc24a73b0c6bb1291412f39",
            "5cda9cab944d4ee69a0b378b1734d6be",
            "e01d67e1125c40b19dc6f2a912b5a0cf",
            "563decc39f364d239bb9af680ecee5d7",
            "fd1db82290f549f19880124ee8e2b7cd",
            "bb214d9c3df04182a21f1edd293ac4a2"
          ]
        },
        "id": "Bn1WXloJyTsM",
        "outputId": "c1bdbd7c-694c-43bf-8b9b-bc83fec8969c"
      },
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModel, pipeline\n",
        "\n",
        "model_name = 'bert-base-uncased'\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "model = AutoModel.from_pretrained(model_name)\n",
        "\n",
        "sentences = [\n",
        "    \"Hey, how are you today?\",\n",
        "    \"I hope you have life-work balances.\",\n",
        "]\n",
        "\n",
        "# tokenize a batch of inputs. \"pt\" means [p]y[t]orch tensors\n",
        "tokens_info = tokenizer(sentences, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "\n",
        "for key in tokens_info:\n",
        "    print(key, tokens_info[key])\n",
        "\n",
        "print(\"Detokenized:\")\n",
        "for i in range(2):\n",
        "    print(tokenizer.decode(tokens_info['input_ids'][i]))\n",
        "\n",
        "# apply the model to get embeddings\n",
        "with torch.no_grad():\n",
        "    token_embeddings, sentence_embedding = model(**tokens_info)\n",
        "\n",
        "print(\"Token: \", token_embeddings)\n",
        "print(\"Sentence: \", sentence_embedding)\n",
        "\n",
        "# See on one sentence:\n",
        "input_sentence = torch.tensor(tokenizer.encode(\"My awesome sentence\")).unsqueeze(0)\n",
        "print(tokenizer.encode(\"My awesome sentence\"))\n",
        "out = model(input_sentence)\n",
        "embeddings_of_last_layer = out[0]\n",
        "print(embeddings_of_last_layer.shape)\n",
        "cls_embeddings = embeddings_of_last_layer[0]\n",
        "# [CLS] \n",
        "cls_embeddings"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "88d325627e0141a1a99d4e5c38538c68"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input_ids tensor([[ 101, 4931, 1010, 2129, 2024, 2017, 2651, 1029,  102,    0,    0,    0],\n",
            "        [ 101, 1045, 3246, 2017, 2031, 2166, 1011, 2147, 5703, 2015, 1012,  102]])\n",
            "token_type_ids tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
            "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
            "attention_mask tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],\n",
            "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])\n",
            "Detokenized:\n",
            "[CLS] hey, how are you today? [SEP] [PAD] [PAD] [PAD]\n",
            "[CLS] i hope you have life - work balances. [SEP]\n",
            "Token:  last_hidden_state\n",
            "Sentence:  pooler_output\n",
            "[101, 2026, 12476, 6251, 102]\n",
            "torch.Size([1, 5, 768])\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.0051,  0.1006,  0.0311,  ..., -0.0059,  0.0928, -0.0599],\n",
              "        [-0.3759, -0.0244,  0.4438,  ..., -0.2665,  0.5087,  0.2046],\n",
              "        [-0.0235, -0.1542, -0.1254,  ...,  0.2556, -0.1367, -0.1162],\n",
              "        [-0.3472, -0.1840, -0.4172,  ...,  0.6273, -0.3594, -0.5472],\n",
              "        [ 0.7600, -0.0073, -0.1210,  ..., -0.0728, -0.7092, -0.3475]],\n",
              "       grad_fn=<SelectBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOIXQ3yuCwoH",
        "outputId": "7dd26ba8-ae0b-4468-dcea-c844340c5188"
      },
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "\n",
        "model_name = 'bert-base-uncased'\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "\n",
        "text = \"After stealing money from the bank vault, the bank robber was seen fishing on the Mississippi river bank.\"\n",
        "\n",
        "marked_text = \"[CLS] \" + text + \" [SEP]\"\n",
        "tokenized_text = tokenizer.tokenize(marked_text)\n",
        "print(tokenized_text)\n",
        "\n",
        "print(list(tokenizer.vocab.keys())[5000:5010])\n",
        "\n",
        "indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "\n",
        "for tup in zip(tokenized_text, indexed_tokens):\n",
        "    print(tup)\n",
        "\n",
        "segments_ids = [1] * len(tokenized_text)\n",
        "print(segments_ids)\n",
        "\n",
        "# Convert inputs to PyTorch tensors\n",
        "tokens_tensor = torch.tensor([indexed_tokens])\n",
        "segments_tensors = torch.tensor([segments_ids])\n",
        "\n",
        "# Load pre-trained model (weights)\n",
        "model = BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True)\n",
        "# Put the model in \"evaluation\" mode, meaning feed-forward operation.\n",
        "model = model.eval()\n",
        "\n",
        "# Predict hidden states features for each layer\n",
        "with torch.no_grad():\n",
        "    out = model(tokens_tensor, segments_tensors)\n",
        "\n",
        "encoded_layers = out[2]"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['[CLS]', 'after', 'stealing', 'money', 'from', 'the', 'bank', 'vault', ',', 'the', 'bank', 'robber', 'was', 'seen', 'fishing', 'on', 'the', 'mississippi', 'river', 'bank', '.', '[SEP]']\n",
            "['matt', 'niece', '[unused446]', 'fell', 'vicky', '##ener', '##rnet', 'flickered', 'disagreement', '[unused18]']\n",
            "('[CLS]', 101)\n",
            "('after', 2044)\n",
            "('stealing', 11065)\n",
            "('money', 2769)\n",
            "('from', 2013)\n",
            "('the', 1996)\n",
            "('bank', 2924)\n",
            "('vault', 11632)\n",
            "(',', 1010)\n",
            "('the', 1996)\n",
            "('bank', 2924)\n",
            "('robber', 27307)\n",
            "('was', 2001)\n",
            "('seen', 2464)\n",
            "('fishing', 5645)\n",
            "('on', 2006)\n",
            "('the', 1996)\n",
            "('mississippi', 5900)\n",
            "('river', 2314)\n",
            "('bank', 2924)\n",
            "('.', 1012)\n",
            "('[SEP]', 102)\n",
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kKbdJ69TDWJf",
        "outputId": "045203d7-d4f7-4e8b-8474-62975a4692e2"
      },
      "source": [
        "layer_i = 0\n",
        "\n",
        "print (\"Number of batches:\", len(encoded_layers[layer_i]))\n",
        "batch_i = 0\n",
        "\n",
        "print (\"Number of tokens:\", len(encoded_layers[layer_i][batch_i]))\n",
        "token_i = 0\n",
        "\n",
        "print (\"Number of hidden units:\", len(encoded_layers[layer_i][batch_i][token_i]))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of batches: 1\n",
            "Number of tokens: 22\n",
            "Number of hidden units: 768\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 592
        },
        "id": "sIR7J0NuDcFJ",
        "outputId": "e269ec41-0566-4a63-b277-2dff65d83f07"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "# For the 5th token in our sentence, select its feature values from layer 5.\n",
        "token_i = 5\n",
        "layer_i = 5\n",
        "vec = encoded_layers[layer_i][batch_i][token_i]\n",
        "\n",
        "# Plot the values as a histogram to show their distribution.\n",
        "plt.figure(figsize=(10,10))\n",
        "plt.hist(vec, bins=200)\n",
        "plt.show()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAI/CAYAAAC4QOfKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVxElEQVR4nO3df4xl93nX8c+DJwkVBdLgqbHimHFVl8qhxEFbK6hFKHaTurjUBtooFYJFWFpRCkogqEwShFQJpE2LmlYI/rDqqAtKSdImwVanQI2bUkDE6To/mjhuiBs2NI4Tb0qipkKkcvPwx1yna++uZ56dH/fuzuslWXPPuXfmPv5qd+a95945p7o7AADs3h9a9gAAAJcbAQUAMCSgAACGBBQAwJCAAgAYElAAAENrh/lkV199dW9sbBzmUwIAXJKHH374C929fqH7DjWgNjY2cvr06cN8SgCAS1JVn77YfV7CAwAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAoDL3MbmVjY2t5Y9xpEioAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABhaW/YAAMD+2Njc+trtMyfvWOIkVz5HoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAobXdPKiqziT5cpLfT/JUdx+rqhcleWeSjSRnkrymu794MGMCAKyOyRGoV3b3zd19bLG9meTB7r4xyYOLbQCAK95eXsK7M8mpxe1TSe7a+zgAAKtvtwHVSX6pqh6uqhOLfdd09xOL259Lcs2+TwcAsIJ29R6oJN/Z3Y9X1TcmeaCqfuPcO7u7q6ov9ImL4DqRJNdff/2ehgUAWAW7OgLV3Y8vPj6Z5L1Jbkny+aq6NkkWH5+8yOfe093HuvvY+vr6/kwNALBEOwZUVf2RqvqjT99O8uokH0tyf5Lji4cdT3LfQQ0JALBKdvMS3jVJ3ltVTz/+Z7v7P1bVryV5V1XdneTTSV5zcGMCAKyOHQOquz+V5GUX2P/bSW47iKEAAFaZM5EDAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQBXoI3NrWxsbu24j0sjoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwtLbsAQCAg+PEmQfDESgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoADjCNja3nGzzEggoAIAhAQUAMCSgAACGBBQAwNCuA6qqrqqqD1XVLyy2b6iqh6rqsap6Z1U9/+DGBABYHZMjUK9L8ug5229J8tbu/uYkX0xy934OBgCwqnYVUFV1XZI7kvz0YruS3Jrk5xcPOZXkroMYEABg1ez2CNRPJvmRJF9dbP+JJF/q7qcW259J8uJ9ng0AYCXtGFBV9b1Jnuzuhy/lCarqRFWdrqrTZ8+evZQvAQAsOPHlatjNEajvSPJ9VXUmyTuy/dLdTyV5YVWtLR5zXZLHL/TJ3X1Pdx/r7mPr6+v7MDIAwHLtGFDd/cbuvq67N5K8Nskvd/dfT/K+JN+/eNjxJPcd2JQAACtkL+eB+sdJ/mFVPZbt90Tduz8jAQCstrWdH/IHuvtXkvzK4vanktyy/yMBAKw2ZyIHABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADC0tuwBAIC5jc2tZY9wpDkCBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhJ9IEgCPmQifhPHffmZN3HOY4lyVHoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACG1pY9AADw3DY2t5Y9As/iCBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIZ2DKiq+sNV9YGq+khVPVJVP7rYf0NVPVRVj1XVO6vq+Qc/LgDA8u3mCNRXktza3S9LcnOS26vqFUnekuSt3f3NSb6Y5O6DGxMAYHXsGFC97XcXm89b/NdJbk3y84v9p5LcdSATAgCsmF29B6qqrqqqDyd5MskDSX4zyZe6+6nFQz6T5MUHMyIAwGrZVUB19+93981JrktyS5Jv3e0TVNWJqjpdVafPnj17iWMCAIdlY3MrG5tbyx5jpY1+C6+7v5TkfUn+fJIXVtXa4q7rkjx+kc+5p7uPdfex9fX1PQ0LALAKdvNbeOtV9cLF7a9L8qokj2Y7pL5/8bDjSe47qCEBAFbJ2s4PybVJTlXVVdkOrnd19y9U1ceTvKOq/lmSDyW59wDnBABYGTsGVHf/epKXX2D/p7L9figAgCPFmcgBAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAKyQjc2tbGxuLXsMdiCgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADC0tuwBAIDzOZnmanMECgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBobdkDAMBRt7G5tewRGHIECgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIZ2DKiqeklVva+qPl5Vj1TV6xb7X1RVD1TVJxcfv+HgxwUAWL7dHIF6KskbuvumJK9I8sNVdVOSzSQPdveNSR5cbAMAXPF2DKjufqK7P7i4/eUkjyZ5cZI7k5xaPOxUkrsOakgAgFUyeg9UVW0keXmSh5Jc091PLO76XJJr9nUyAIAVtbbbB1bV1yd5d5LXd/fvVNXX7uvurqq+yOedSHIiSa6//vq9TQsAHJqNza2v3T5z8o4lTrJ6dnUEqqqel+14ent3v2ex+/NVde3i/muTPHmhz+3ue7r7WHcfW19f34+ZAQCWaje/hVdJ7k3yaHf/xDl33Z/k+OL28ST37f94AACrZzcv4X1Hkr+R5KNV9eHFvjclOZnkXVV1d5JPJ3nNwYwIALBadgyo7v5vSeoid9+2v+MAAKw+ZyIHABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAobVlDwAArL6Nza3z9p05eccSJlkNjkABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwNDasgcAgKNqY3Nr2SPsybnznzl5xxInOXyOQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgyIk0AeCAXO4nypy40P/rlXxyTUegAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYGjHgKqqt1XVk1X1sXP2vaiqHqiqTy4+fsPBjgkAsDp2cwTqZ5Lc/qx9m0ke7O4bkzy42AYAOBJ2DKju/tUk/+dZu+9Mcmpx+1SSu/Z5LgCAlXWp74G6prufWNz+XJJr9mkeAICVt+c3kXd3J+mL3V9VJ6rqdFWdPnv27F6fDgBg6S41oD5fVdcmyeLjkxd7YHff093HuvvY+vr6JT4dAMDquNSAuj/J8cXt40nu259xAABW325OY/DvkvyPJH+6qj5TVXcnOZnkVVX1ySTftdgGADgS1nZ6QHf/4EXuum2fZwEAuCw4EzkAwJCAAgAYElAAAEMCCgBgaMc3kQMAMxubW8segQPmCBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAHAgNja3rtiTigooAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAytLXsAALjcnHtyyDMn71jiJCyLI1AAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGHIiTQDgQF2JJx51BAoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQ06kCQAcusv95JqOQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABwB5sbG4946SQzF2OayigAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhtaWPQAAcHRMzzh+7uPPnLxjv8e5ZI5AAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGDIiTQB4Dns9kSO0xNE8txWfT0dgQIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADA0BV3Is3dnvAMAC7mYidxXPWTO17pnl7/Vfj57ggUAMCQgAIAGBJQAABDAgoAYGhPAVVVt1fVJ6rqsara3K+hAABW2SUHVFVdleRfJfmeJDcl+cGqumm/BgMAWFV7OQJ1S5LHuvtT3f17Sd6R5M79GQsAYHXtJaBenOS3ztn+zGIfAMAV7cBPpFlVJ5KcWGz+blV94qCf82vP/ZbDeqaRq5N8YdlDrBhrcj5rcj5rcj5rcj5r8kyX1Xrs9uf2Hn++T9bkT13sjr0E1ONJXnLO9nWLfc/Q3fckuWcPz3NFqarT3X1s2XOsEmtyPmtyPmtyPmtyPmvyTNbjfPu1Jnt5Ce/XktxYVTdU1fOTvDbJ/XsdCABg1V3yEajufqqq/l6S/5TkqiRv6+5H9m0yAIAVtaf3QHX3Lyb5xX2a5ajwcub5rMn5rMn5rMn5rMn5rMkzWY/z7cuaVHfvx9cBADgyXMoFAGBIQB2SqvqBqnqkqr5aVcfO2f+qqnq4qj66+HjrMuc8TBdbk8V9b1xcIugTVfXdy5pxmarq5qp6f1V9uKpOV9Uty55p2arq71fVbyz+3PzYsudZFVX1hqrqqrp62bMsW1X9+OLPyK9X1Xur6oXLnmlZXG7tmarqJVX1vqr6+OJ7yOv28vUE1OH5WJK/muRXn7X/C0n+cnd/W5LjSf7tYQ+2RBdck8UlgV6b5KVJbk/yrxeXDjpqfizJj3b3zUn+6WL7yKqqV2b7agcv6+6XJvkXSx5pJVTVS5K8Osn/XvYsK+KBJH+mu/9skv+Z5I1LnmcpXG7tgp5K8obuvinJK5L88F7WREAdku5+tLvPO4lod3+ouz+72HwkyddV1QsOd7rluNiaZPuH5Du6+yvd/b+SPJbtSwcdNZ3kjy1u//Ekn32Oxx4FP5TkZHd/JUm6+8klz7Mq3prkR7L95+XI6+5f6u6nFpvvz/Y5Co8il1t7lu5+ors/uLj95SSPZg9XUBFQq+WvJfng0z8gjjCXCdr2+iQ/XlW/le2jLUfyX9Ln+JYkf6GqHqqq/1JV377sgZatqu5M8nh3f2TZs6yov53kPyx7iCXxffQ5VNVGkpcneehSv8aBX8rlKKmq/5zkT17grjd39307fO5Lk7wl24firxh7WZOj4LnWJ8ltSf5Bd7+7ql6T5N4k33WY8x22HdZjLcmLsn3o/duTvKuqvqmv8F8l3mFN3pQr7HvGbuzm+0pVvTnbL9m8/TBnY/VV1dcneXeS13f371zq1xFQ+6i7L+mHW1Vdl+S9Sf5md//m/k61XJe4Jru6TNCV4LnWp6r+TZKn3+T4c0l++lCGWqId1uOHkrxnEUwfqKqvZvuaVmcPa75luNiaVNW3JbkhyUeqKtn+e/LBqrqluz93iCMeup2+r1TV30ryvUluu9ID+zkcme+jE1X1vGzH09u7+z17+VpewluyxW+IbCXZ7O7/vux5VsT9SV5bVS+oqhuS3JjkA0ueaRk+m+QvLm7fmuSTS5xlFfz7JK9Mkqr6liTPz2V0kdT91t0f7e5v7O6N7t7I9ks0f+5Kj6edVNXt2X5P2Pd19/9d9jxL5HJrz1Lb/9K4N8mj3f0Te/56RzfOD1dV/ZUk/zLJepIvJflwd393Vf2TbL+35dwfjq8+Cm+QvdiaLO57c7bfv/BUtg+zHrn3MVTVdyb5qWwfKf5/Sf5udz+83KmWZ/FD4G1Jbk7ye0n+UXf/8nKnWh1VdSbJse4+slGZJFX1WJIXJPntxa73d/ffWeJIS1NVfynJT+YPLrf2z5c80lItvqf+1yQfTfLVxe43La6qMv96AgoAYMZLeAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAY+v8yrajMccmJFwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iSSkjwWLDoSU",
        "outputId": "4c84706e-eea0-44ec-f348-7f74099b6ed6"
      },
      "source": [
        "# Convert the hidden state embeddings into single token vectors\n",
        "\n",
        "# Holds the list of 12 layer embeddings for each token\n",
        "# Will have the shape: [# tokens, # layers, # features]\n",
        "token_embeddings = [] \n",
        "\n",
        "# For each token in the sentence...\n",
        "for token_i in range(len(tokenized_text)):\n",
        "  # Holds 12 layers of hidden states for each token \n",
        "    hidden_layers = [] \n",
        "  \n",
        "  # For each of the 12 layers...\n",
        "    for layer_i in range(len(encoded_layers)):\n",
        "    \n",
        "    # Lookup the vector for `token_i` in `layer_i`\n",
        "        vec = encoded_layers[layer_i][batch_i][token_i]\n",
        "        hidden_layers.append(vec)\n",
        "    \n",
        "    token_embeddings.append(hidden_layers)\n",
        "\n",
        "# Sanity check the dimensions:\n",
        "print (\"Number of tokens in sequence:\", len(token_embeddings))\n",
        "print (\"Number of layers per token:\", len(token_embeddings[0]))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of tokens in sequence: 22\n",
            "Number of layers per token: 13\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yGuPEQvMDqwV",
        "outputId": "c5a855ff-7e87-47d0-a5ce-f14c89cd566d"
      },
      "source": [
        "concatenated_last_4_layers = [torch.cat((layer[-1], layer[-2], layer[-3], layer[-4]), 0) for layer in token_embeddings] # [number_of_tokens, 3072]\n",
        "\n",
        "summed_last_4_layers = [torch.sum(torch.stack(layer)[-4:], 0) for layer in token_embeddings] # [number_of_tokens, 768]\n",
        "sentence_embedding = torch.mean(encoded_layers[11], 1)\n",
        "print (\"Our final sentence embedding vector of shape:\"), sentence_embedding[0].shape[0]"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Our final sentence embedding vector of shape:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(None, 768)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FPqkaJFiD31Q",
        "outputId": "ee0d093d-6f4a-4f7f-d9d2-7529de7b36e2"
      },
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "\n",
        "# Compare \"bank\" as in \"bank robber\" to \"bank\" as in \"river bank\"\n",
        "different_bank = cosine_similarity(summed_last_4_layers[10].reshape(1,-1), summed_last_4_layers[19].reshape(1,-1))[0][0]\n",
        "\n",
        "# Compare \"bank\" as in \"bank robber\" to \"bank\" as in \"bank vault\" \n",
        "same_bank = cosine_similarity(summed_last_4_layers[10].reshape(1,-1), summed_last_4_layers[6].reshape(1,-1))[0][0]\n",
        "\n",
        "print(\"Similarity of 'bank' as in 'bank robber' to 'bank' as in 'bank vault':\",  same_bank)\n",
        "print (\"Similarity of 'bank' as in 'bank robber' to 'bank' as in 'river bank':\",  different_bank)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Similarity of 'bank' as in 'bank robber' to 'bank' as in 'bank vault': 0.9386391\n",
            "Similarity of 'bank' as in 'bank robber' to 'bank' as in 'river bank': 0.6932362\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://raw.githubusercontent.com/king-menin/mipt-nlp2022/master/seminars/sem6/train.jsonl"
      ],
      "metadata": {
        "id": "AZy_rkb7ww5F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://raw.githubusercontent.com/king-menin/mipt-nlp2022/master/seminars/sem6/val.jsonl"
      ],
      "metadata": {
        "id": "RReKu_RexITq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tLmCfm-i6NPg"
      },
      "source": [
        "##### Fine tune BERT on classification task"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aPqWNq50yTul"
      },
      "source": [
        "# !pip install transformers==4.0.1\n",
        "# !pip install sentencepiece==0.1.94\n",
        "import random\n",
        "import torch\n",
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification, AdamW, AutoModelForSeq2SeqLM"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MAxr0T7GH4Pe"
      },
      "source": [
        "# Use a GPU if you have one available (Runtime -> Change runtime type -> GPU)\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Set seeds for reproducibility\n",
        "random.seed(26)\n",
        "np.random.seed(26)\n",
        "torch.manual_seed(26)\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"DeepPavlov/rubert-base-cased\")\n",
        "model = AutoModelForSequenceClassification.from_pretrained(\"DeepPavlov/rubert-base-cased\")\n",
        "model.to(device)\n",
        "\n",
        "learning_rate = 1e-5\n",
        "optimizer = AdamW(model.parameters(), lr=learning_rate, eps=1e-8)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iawSE9-YH4SA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fc91c0c4-33b1-4a26-b181-657b4c62c44d"
      },
      "source": [
        "def encode_data(tokenizer, questions, passages, max_length):\n",
        "    \"\"\"Encode the question/passage pairs into features than can be fed to the model.\"\"\"\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    for question, passage in zip(questions, passages):\n",
        "        encoded_data = tokenizer.encode_plus(question, passage, max_length=max_length, pad_to_max_length=True, truncation_strategy=\"longest_first\")\n",
        "        encoded_pair = encoded_data[\"input_ids\"]\n",
        "        attention_mask = encoded_data[\"attention_mask\"]\n",
        "\n",
        "        input_ids.append(encoded_pair)\n",
        "        attention_masks.append(attention_mask)\n",
        "\n",
        "    return np.array(input_ids), np.array(attention_masks)\n",
        "\n",
        "# Loading data\n",
        "train_data_df = pd.read_json(\"train.jsonl\", lines=True, orient='records')\n",
        "dev_data_df = pd.read_json(\"val.jsonl\", lines=True, orient=\"records\")\n",
        "\n",
        "passages_train = train_data_df.passage.values\n",
        "questions_train = train_data_df.question.values\n",
        "answers_train = [1 if ans else 0 for ans in train_data_df.label.values]\n",
        "answers_train = train_data_df.label.values.astype(int)\n",
        "\n",
        "passages_dev = dev_data_df.passage.values\n",
        "questions_dev = dev_data_df.question.values\n",
        "answers_dev = [1 if ans else 0 for ans in dev_data_df.label.values]\n",
        "\n",
        "# Encoding data\n",
        "max_seq_length = 256\n",
        "input_ids_train, attention_masks_train = encode_data(tokenizer, questions_train, passages_train, max_seq_length)\n",
        "input_ids_dev, attention_masks_dev = encode_data(tokenizer, questions_dev, passages_dev, max_seq_length)\n",
        "\n",
        "train_features = (input_ids_train, attention_masks_train, answers_train)\n",
        "dev_features = (input_ids_dev, attention_masks_dev, answers_dev)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2277: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E8AEwrVnH_LI"
      },
      "source": [
        "# Building Dataloaders\n",
        "batch_size = 4\n",
        "\n",
        "train_features_tensors = [torch.tensor(feature, dtype=torch.long) for feature in train_features]\n",
        "dev_features_tensors = [torch.tensor(feature, dtype=torch.long) for feature in dev_features]\n",
        "\n",
        "train_dataset = TensorDataset(*train_features_tensors)\n",
        "dev_dataset = TensorDataset(*dev_features_tensors)\n",
        "\n",
        "train_sampler = RandomSampler(train_dataset)\n",
        "dev_sampler = SequentialSampler(dev_dataset)\n",
        "\n",
        "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch_size)\n",
        "dev_dataloader = DataLoader(dev_dataset, sampler=dev_sampler, batch_size=batch_size)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YbjjBj-eIBO6"
      },
      "source": [
        "epochs = 10\n",
        "grad_acc_steps = 1\n",
        "train_loss_values = []\n",
        "dev_acc_values = []\n",
        "\n",
        "for epoch in tqdm(range(epochs), desc=\"Epoch\"):\n",
        "  # Training\n",
        "  epoch_train_loss = 0 # Cumulative loss\n",
        "  model.train()\n",
        "  model.zero_grad()\n",
        "\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "      input_ids = batch[0].to(device)\n",
        "      attention_masks = batch[1].to(device)\n",
        "      labels = batch[2].to(device)     \n",
        "\n",
        "      outputs = model(input_ids, attention_mask=attention_masks, labels=labels)\n",
        "\n",
        "      loss = outputs[0]\n",
        "      loss = loss / grad_acc_steps\n",
        "      epoch_train_loss += loss.item()\n",
        "\n",
        "      loss.backward()\n",
        "      \n",
        "      if (step+1) % grad_acc_steps == 0: # Gradient accumulation is over\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) # Clipping gradients\n",
        "        optimizer.step()\n",
        "        model.zero_grad()\n",
        "\n",
        "  epoch_train_loss = epoch_train_loss / len(train_dataloader)          \n",
        "  train_loss_values.append(epoch_train_loss)\n",
        "  \n",
        "  # Evaluation\n",
        "  epoch_dev_accuracy = 0 # Cumulative accuracy\n",
        "  model.eval()\n",
        "\n",
        "  for batch in dev_dataloader:\n",
        "    \n",
        "    input_ids = batch[0].to(device)\n",
        "    attention_masks = batch[1].to(device)\n",
        "    labels = batch[2]\n",
        "    \n",
        "    with torch.no_grad():        \n",
        "        outputs = model(input_ids, attention_mask=attention_masks)\n",
        "                    \n",
        "    logits = outputs[0]\n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    \n",
        "    predictions = np.argmax(logits, axis=1).flatten()\n",
        "    labels = labels.numpy().flatten()\n",
        "    \n",
        "    epoch_dev_accuracy += np.sum(predictions == labels) / len(labels)\n",
        "\n",
        "  epoch_dev_accuracy = epoch_dev_accuracy / len(dev_dataloader)\n",
        "  print('Epoch', epoch)\n",
        "  print('Accuracy:', round(epoch_dev_accuracy, 4),'\\n')\n",
        "  dev_acc_values.append(epoch_dev_accuracy)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s71RUzXhyUuZ"
      },
      "source": [
        "### GPT3 examples Russian\n",
        "\n",
        "You can test, generate and finetune Russian GPT2,3 models\n",
        "See them [here](https://github.com/sberbank-ai/ru-gpts)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rgmJnZvgyTwp",
        "outputId": "873de567-bf38-4a49-a9fb-f7bdc3334052"
      },
      "source": [
        "!git clone  https://github.com/sberbank-ai/ru-gpts"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'ru-gpts' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8TtUQ650yTzE",
        "outputId": "414c91f3-7a95-49a3-8a29-a027e59f060c"
      },
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "np.random.seed(42)\n",
        "torch.manual_seed(42)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7f900c9fddb0>"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l_t0C8YtycAW"
      },
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Tokenizer"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QTGv6MDwynqI"
      },
      "source": [
        "Some usefull parameters of generation:\n",
        "\n",
        "* top_p - if set to float < 1, only the most probable tokens with probabilities that add up to top_p or higher are kept for generation.\n",
        "* top_k - the number of highest probability vocabulary tokens to keep for top-k-filtering\n",
        "* temperature - value used to module the next token probabilities\n",
        "\n",
        "You can always change decoding methods: Greedy search, Beam search, Top-K sampling* and Top-p sampling.\n",
        "\n",
        "*sampling means randomly picking the next word w_t according to its conditional probability distribution.\n",
        "\n",
        "Read about different inference techniques https://huggingface.co/blog/how-to-generate and parameters.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BhJGTFSRyduZ"
      },
      "source": [
        "def load_tokenizer_and_model(model_name_or_path):\n",
        "  return GPT2Tokenizer.from_pretrained(model_name_or_path), GPT2LMHeadModel.from_pretrained(model_name_or_path).cuda()\n",
        "\n",
        "\n",
        "def generate(\n",
        "    model, tok, text,\n",
        "    do_sample=True, max_length=100, repetition_penalty=1.5,\n",
        "    top_k=10, top_p=0.95, temperature=1,\n",
        "    num_beams=None,\n",
        "    no_repeat_ngram_size=3\n",
        "    ):\n",
        "  input_ids = tok.encode(text, return_tensors=\"pt\").cuda()\n",
        "  out = model.generate(\n",
        "      input_ids.cuda(),\n",
        "      max_length=max_length,\n",
        "      repetition_penalty=repetition_penalty,\n",
        "      do_sample=do_sample,\n",
        "      top_k=top_k, top_p=top_p, temperature=temperature,\n",
        "      num_beams=num_beams, no_repeat_ngram_size=no_repeat_ngram_size\n",
        "      )\n",
        "  return list(map(tok.decode, out))"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198,
          "referenced_widgets": [
            "e00059f36bc145d3a05b59d521c76daa",
            "95ae4bcb7dbf45108671d4dbdf8cc386",
            "a4a084e20a8045db8f3879f6edc22d52",
            "07a3fdc2c5a049aaabc710e28091d9fa",
            "e884f6203b8f4f6abba02bec8cd9af95",
            "98929f9f516f44f6a16795ea55e1caf0",
            "8cd43912df8b4d5cad13d02897487432",
            "6a459cd9d0b34884b90d7e027dbf400e",
            "71613e16288a480194b86d2e9ecc71bc",
            "2aa303232ae147db8425a2bf9bb659b2",
            "3fe62ae12da4439a88c042e22d69b103",
            "0808d90bc4a243bb80f36ea3f7d7b062",
            "c1df4250aeb04179a83287bd420dee7d",
            "f9a2cb6c139c40f992996f7d2aac9880",
            "b86386b7cbd44c31bee16b81efef0531",
            "fd38ef412721463cbdabba5cebafa304",
            "1adbefab111244d8b198b4f062dc1ccb",
            "618aebc4a96e4ecd825d209aea007024",
            "14aae45587144a19933d9c2dab2a2f42",
            "8daf1a40a271418bb12ac88de1890159",
            "0cc2b7655ca74a6d9f36a679d33c562b",
            "25ce556509dd48949920ced5491cdec0",
            "a0ff84a01d0e49a4878ceee491c21b74",
            "600034afb5dc45dda6a848e53f84c5fe",
            "f010200682c14090a43885f381b59bff",
            "c46452c83ad84ae882fcf286905992c5",
            "ebc162eb8af84b42a53cbd4f853507e6",
            "f3ae323fda034811a84ecf485ada86c8",
            "7b27ce6d076645c1a638ed90ffb7e589",
            "9f723e627a9746eba28fc6e35fa1ed0b",
            "9a3e4f11cdf244a388ed18aff392bccf",
            "e280f6adbbf44e2e8ad49fb33a831998",
            "413157f320f64f829e14c8224e5a2e88",
            "b266ea1ffe8f4df09fa167391df40fa8",
            "a32d661df906474a88a97f4f88d51de9",
            "2141112c8ea44b618067066ebdf186ff",
            "80fc8d42c19149959d5d2ad69956929c",
            "18ddc2ddde6b49ae8fbee85ac0d7af24",
            "d99edbd5725c417386f39d7d83d8bcac",
            "ced35c4752874f8e9307828a2454552b",
            "2a9534fb470d47439c9eaeb9b6cffcbb",
            "1c6667c42d9644228a974a013acc39ca",
            "88633d1e40ef4b1b87295954e1f74636",
            "7e1abd404a074de8afcb1f8d47a789f8"
          ]
        },
        "id": "B5VE6DdSyfyw",
        "outputId": "1a269224-d331-4bd6-b1ab-d1bf3ba62a4f"
      },
      "source": [
        "tokenizer, model = load_tokenizer_and_model(\"sberbank-ai/rugpt3large_based_on_gpt2\")\n",
        "generated = generate(model, tokenizer, \"На словах ты Лев Толстой, а  \", num_beams=10)\n",
        "generated[0]"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.63M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e00059f36bc145d3a05b59d521c76daa"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.21M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0808d90bc4a243bb80f36ea3f7d7b062"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/609 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a0ff84a01d0e49a4878ceee491c21b74"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/2.93G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b266ea1ffe8f4df09fa167391df40fa8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'На словах ты Лев Толстой, а   на  деле  -  какой-нибудь  мелкий  чиновник.  Но  это  не  важно.  Важно  то,  что  у  тебя  есть  деньги,  и  ты  их  тратишь.  Ты  можешь  купить  себе  новую  машину,  но  для  того,  чтобы  ее  купить,  тебе  нужны  деньги.  Вот  почему  я  так '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "u8EBaWsY1b5v",
        "outputId": "670dae32-9389-4267-979a-c5493554352a"
      },
      "source": [
        "generated = generate(model, tokenizer, \n",
        "  \"Кошка => Кошки\\n\\\n",
        "  Собака => Собаки\\n\\\n",
        "  Коза => Козы\\n\\\n",
        "  Овца => Овцы\\n \\\n",
        "  Петух => Петухи\\n \\\n",
        "  Волк => Волки\\n \\\n",
        "  Зебра => \", num_beams=2\n",
        ")\n",
        "generated[0]"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Кошка => Кошки\\n  Собака => Собаки\\n  Коза => Козы\\n  Овца => Овцы\\n   Петух => Петухи\\n   Волк => Волки\\n   Зебра =>  Зебры\\n  Змея => Змеи\\n  Олень => Олени\\n  Зайчик => Зайчики\\n  Лошадка => Лошадки\\n  Курица => Курочки\\n  Лягушка'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    }
  ]
}