llama2-0.93.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"machine_shape":"hm","gpuType":"A100","authorship_tag":"ABX9TyNnmoKPnM/mX1kemE0JcUmt"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"40adaf0c426d44a6a55ad54bc40d54cd":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_fd4834fde88d454b9e91656e368f7415","IPY_MODEL_b0853520b45a4d58b01327a8291fd1d0","IPY_MODEL_3790b42605e04c9c80ad81f4c699e6e8"],"layout":"IPY_MODEL_9447c0523d82433ba3ec2006dc466f97"}},"fd4834fde88d454b9e91656e368f7415":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_cb65db54b02349eea423529b78913c77","placeholder":"​","style":"IPY_MODEL_c11fdfb4d20c4cef93940be6d9dfa763","value":"Map: 100%"}},"b0853520b45a4d58b01327a8291fd1d0":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9d99576fc3694d49a67f15b005bb4a91","max":67349,"min":0,"orientation":"horizontal","style":"IPY_MODEL_69f55c1604f748048a4bdcaa512cff2e","value":67349}},"3790b42605e04c9c80ad81f4c699e6e8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_3517fdbd89f44b7599de6f263a9f3cb3","placeholder":"​","style":"IPY_MODEL_8d9e98fd07214cf989393d760c8072c6","value":" 67349/67349 [00:04&lt;00:00, 17707.87 examples/s]"}},"9447c0523d82433ba3ec2006dc466f97":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cb65db54b02349eea423529b78913c77":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c11fdfb4d20c4cef93940be6d9dfa763":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9d99576fc3694d49a67f15b005bb4a91":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"69f55c1604f748048a4bdcaa512cff2e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"3517fdbd89f44b7599de6f263a9f3cb3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8d9e98fd07214cf989393d760c8072c6":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c30b7778d54349fcb575452d602598be":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7451dbe390564162ae1b8614909ab1b3","IPY_MODEL_d8d157a104f44cf9b2e718621d30d37e","IPY_MODEL_599f251afcd44b558fa89b2c245340da"],"layout":"IPY_MODEL_48b6c572da6e43a2a16dd17d11a9c159"}},"7451dbe390564162ae1b8614909ab1b3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f0d8545ced23468a8a1d15a0533c7d73","placeholder":"​","style":"IPY_MODEL_648c2eac59754a4ba4c0ba756f7da2a8","value":"Map: 100%"}},"d8d157a104f44cf9b2e718621d30d37e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_ca8bbf77d9644296b288e58ef9b8ed1f","max":872,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f72771dbfb12428b99ea3bf9bbd6190a","value":872}},"599f251afcd44b558fa89b2c245340da":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8d4678b1f8754bc88b527c799753bcdf","placeholder":"​","style":"IPY_MODEL_a1c83cf29fc94f088e4e89b3dcd1050a","value":" 872/872 [00:00&lt;00:00, 12810.44 examples/s]"}},"48b6c572da6e43a2a16dd17d11a9c159":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f0d8545ced23468a8a1d15a0533c7d73":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"648c2eac59754a4ba4c0ba756f7da2a8":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ca8bbf77d9644296b288e58ef9b8ed1f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f72771dbfb12428b99ea3bf9bbd6190a":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8d4678b1f8754bc88b527c799753bcdf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a1c83cf29fc94f088e4e89b3dcd1050a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"940ee807d03e446faf794b3b2c1141f7":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_91c5ddd903da4d41b927a4a3499fb8fb","IPY_MODEL_a275c8a83fe146c89777b4eac0c7a981","IPY_MODEL_77f6699f70bb46a3b9ce502d85700bbd"],"layout":"IPY_MODEL_64421d35dde24620b0c138d8a66d8f57"}},"91c5ddd903da4d41b927a4a3499fb8fb":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8954a12d81c34d33a6869b89d07b3cac","placeholder":"​","style":"IPY_MODEL_d52b1d3a27d94cb6b931d14067700df4","value":"Map: 100%"}},"a275c8a83fe146c89777b4eac0c7a981":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_3b58d94c598a4950ab505c47943b6718","max":67349,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f6d1c92dc0344d2fb96d4aa2fdbb40ad","value":67349}},"77f6699f70bb46a3b9ce502d85700bbd":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b93835517bb6494e99a78348637e1ce8","placeholder":"​","style":"IPY_MODEL_a1b322c229be404294b529211de4897d","value":" 67349/67349 [00:04&lt;00:00, 16649.15 examples/s]"}},"64421d35dde24620b0c138d8a66d8f57":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8954a12d81c34d33a6869b89d07b3cac":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d52b1d3a27d94cb6b931d14067700df4":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"3b58d94c598a4950ab505c47943b6718":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f6d1c92dc0344d2fb96d4aa2fdbb40ad":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"b93835517bb6494e99a78348637e1ce8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a1b322c229be404294b529211de4897d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"419d95d7651a42e0927df97a5bf9d790":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_20d0b401213249899c5e389a01326c02","IPY_MODEL_d12cbc41e2ff4d07a74b6ba83afbe93a","IPY_MODEL_7810ce55366340dbb424bf43df204984"],"layout":"IPY_MODEL_90e24b11099a4d46affa869530f0af39"}},"20d0b401213249899c5e389a01326c02":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9d1092f3832c4cedb2a9fb6e7a102061","placeholder":"​","style":"IPY_MODEL_327dda7df82042a884adfcbffa579792","value":"Map: 100%"}},"d12cbc41e2ff4d07a74b6ba83afbe93a":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_4fc0a7eb081b484199f92314eea182a7","max":872,"min":0,"orientation":"horizontal","style":"IPY_MODEL_b2487423ce1f4a0e82417527234b7042","value":872}},"7810ce55366340dbb424bf43df204984":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d22cc06bdae440deaa865f435e0c178f","placeholder":"​","style":"IPY_MODEL_50d3bccded7748f0a138898a348299a3","value":" 872/872 [00:00&lt;00:00, 12435.64 examples/s]"}},"90e24b11099a4d46affa869530f0af39":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9d1092f3832c4cedb2a9fb6e7a102061":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"327dda7df82042a884adfcbffa579792":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4fc0a7eb081b484199f92314eea182a7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b2487423ce1f4a0e82417527234b7042":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"d22cc06bdae440deaa865f435e0c178f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"50d3bccded7748f0a138898a348299a3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f9aa60d6d9344e0b9ee8e0738326de6a":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_b50c408c1de6450a83902f8d62ba8baa","IPY_MODEL_6e1042c4dc894cdb82ee0a604c584b39","IPY_MODEL_03b6de34bf5e4b879ed3e757c2c4c5e7"],"layout":"IPY_MODEL_4d8e40a0cd8840ff83f34db2ef63f2f4"}},"b50c408c1de6450a83902f8d62ba8baa":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b443e2e670be435a8fc7453fe85f2575","placeholder":"​","style":"IPY_MODEL_8489d803670347ef89988ed8022ca548","value":"Loading checkpoint shards: 100%"}},"6e1042c4dc894cdb82ee0a604c584b39":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_a64f3155b6604a62887ffc907c0d0d17","max":2,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3d359e425e6749e1a2ec2ba8a73df35b","value":2}},"03b6de34bf5e4b879ed3e757c2c4c5e7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8b4e02adbdd74acea3ae47fae62ab9d8","placeholder":"​","style":"IPY_MODEL_916039035b934443ad938283152bf5ab","value":" 2/2 [00:04&lt;00:00,  2.18s/it]"}},"4d8e40a0cd8840ff83f34db2ef63f2f4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b443e2e670be435a8fc7453fe85f2575":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8489d803670347ef89988ed8022ca548":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a64f3155b6604a62887ffc907c0d0d17":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3d359e425e6749e1a2ec2ba8a73df35b":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8b4e02adbdd74acea3ae47fae62ab9d8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"916039035b934443ad938283152bf5ab":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"code","source":["!pip install peft huggingface_hub\n","!pip install -q -U trl transformers accelerate git+https://github.com/huggingface/peft.git\n","!pip install -q datasets bitsandbytes"],"metadata":{"id":"PYk9IPloFL07"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!huggingface-cli login"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"KtJ56g2rO7Gs","executionInfo":{"status":"ok","timestamp":1692294249811,"user_tz":-120,"elapsed":17637,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"54937f01-a0f2-41f6-c37f-0ee76093fef5"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","    _|    _|  _|    _|    _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|_|_|_|    _|_|      _|_|_|  _|_|_|_|\n","    _|    _|  _|    _|  _|        _|          _|    _|_|    _|  _|            _|        _|    _|  _|        _|\n","    _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|\n","    _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|\n","    _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|\n","    \n","    To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n","Token: \n","Add token as git credential? (Y/n) n\n","Token is valid (permission: read).\n","Your token has been saved to /root/.cache/huggingface/token\n","Login successful\n"]}]},{"cell_type":"code","source":["from datasets import load_dataset, DatasetDict\n","from datasets import load_dataset\n","\n","from transformers import AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer, TrainingArguments, pipeline\n","from peft import LoraConfig\n","from trl import SFTTrainer\n","\n","import torch\n","\n","import re\n","from tqdm import tqdm"],"metadata":{"id":"hInW4FqCFwwn","executionInfo":{"status":"ok","timestamp":1692298175429,"user_tz":-120,"elapsed":7,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":35,"outputs":[]},{"cell_type":"code","source":["# Load Dataset\n","\n","raw_train_sets = load_dataset(\"glue\", \"sst2\", split=\"train\")\n","raw_val_sets = load_dataset('glue', 'sst2', split='validation')"],"metadata":{"id":"cCX6XDYhFdW5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def create_label_text(label):\n","    label_map = {\n","        0 : 'negative',\n","        1 : 'positive',\n","    }\n","\n","    return {'label_text': label_map[label]}"],"metadata":{"id":"UyvhDXMAGCEo","executionInfo":{"status":"ok","timestamp":1692294270557,"user_tz":-120,"elapsed":4,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":6,"outputs":[]},{"cell_type":"markdown","source":["# Data pro-pocessing\n","We need to modify sentence's labels to {0 : 'negative', 1 : 'positive'}. Because in prompt-tuning, the vocabulary is easier for model to learn.\n"],"metadata":{"id":"Cn40R62C3SFm"}},{"cell_type":"code","source":["# Convert digital labels to text labels\n","train_sets = raw_train_sets.map(create_label_text, input_columns=['label'])\n","val_sets = raw_val_sets.map(create_label_text, input_columns=['label'])\n","train_sets"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":149,"referenced_widgets":["40adaf0c426d44a6a55ad54bc40d54cd","fd4834fde88d454b9e91656e368f7415","b0853520b45a4d58b01327a8291fd1d0","3790b42605e04c9c80ad81f4c699e6e8","9447c0523d82433ba3ec2006dc466f97","cb65db54b02349eea423529b78913c77","c11fdfb4d20c4cef93940be6d9dfa763","9d99576fc3694d49a67f15b005bb4a91","69f55c1604f748048a4bdcaa512cff2e","3517fdbd89f44b7599de6f263a9f3cb3","8d9e98fd07214cf989393d760c8072c6","c30b7778d54349fcb575452d602598be","7451dbe390564162ae1b8614909ab1b3","d8d157a104f44cf9b2e718621d30d37e","599f251afcd44b558fa89b2c245340da","48b6c572da6e43a2a16dd17d11a9c159","f0d8545ced23468a8a1d15a0533c7d73","648c2eac59754a4ba4c0ba756f7da2a8","ca8bbf77d9644296b288e58ef9b8ed1f","f72771dbfb12428b99ea3bf9bbd6190a","8d4678b1f8754bc88b527c799753bcdf","a1c83cf29fc94f088e4e89b3dcd1050a"]},"id":"n98JapeZGTbv","executionInfo":{"status":"ok","timestamp":1692294278639,"user_tz":-120,"elapsed":4263,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"25eba735-3227-43d9-94d7-3d76ebe10dd8"},"execution_count":7,"outputs":[{"output_type":"display_data","data":{"text/plain":["Map:   0%|          | 0/67349 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"40adaf0c426d44a6a55ad54bc40d54cd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Map:   0%|          | 0/872 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c30b7778d54349fcb575452d602598be"}},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["Dataset({\n","    features: ['sentence', 'label', 'idx', 'label_text'],\n","    num_rows: 67349\n","})"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["train_sets[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5w5qmvt0GWED","executionInfo":{"status":"ok","timestamp":1692294278640,"user_tz":-120,"elapsed":12,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"44ebe015-f70f-4156-8edb-549af706771e"},"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'sentence': 'hide new secretions from the parental units ',\n"," 'label': 0,\n"," 'idx': 0,\n"," 'label_text': 'negative'}"]},"metadata":{},"execution_count":8}]},{"cell_type":"code","source":["def convert_into_prompt_template(system_prompt, user_message, train=True, label_text=\"\"):\n","    # Convert the dataset into the prompt template format as follows:\n","    # <s>[INST] <<SYS>>\n","    # {{ system_prompt }}\n","    # <</SYS>>\n","\n","    # {{ user_message }} [/INST]\n","    # Sentiment: {{ label }} </s>      ## label is neccessary for training, but not for inference\n","    if train:\n","        text = f\"<s>[INST] <<SYS>>\\n{system_prompt}\\n<</SYS>>\\n\\nSentence: {user_message} [/INST]\\nSentiment: {label_text} </s>\"\n","    else:\n","        text = f\"<s>[INST] <<SYS>>\\n{system_prompt}\\n<</SYS>>\\n\\nSentence: {user_message} [/INST]\\nSentiment: \"\n","\n","    return text"],"metadata":{"id":"RlIbfJWgGmhi","executionInfo":{"status":"ok","timestamp":1692294278641,"user_tz":-120,"elapsed":10,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":9,"outputs":[]},{"cell_type":"markdown","source":["Create the system prompt and user prompt for prompt-tuning."],"metadata":{"id":"FNYu0MkG35E0"}},{"cell_type":"code","source":["system_prompt = \"You are a helpful, respectful and honest sentiment analysis assistant. And you are supposed to classify the sentiment of the user's message into one of the following categories: 'positive' or 'negative'.\"\n","user_prompt = \"Classify the sentiment of the following sentence into one of the following categories: positive or negative.\""],"metadata":{"id":"LPzS727zG7oz","executionInfo":{"status":"ok","timestamp":1692296286609,"user_tz":-120,"elapsed":6,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":29,"outputs":[]},{"cell_type":"code","source":["def map_dataset(system_prompt, dataset, train=True):\n","    # Convert the dataset into the format required by the model\n","    def convert(sentence, label_text):\n","        converted_inputs = convert_into_prompt_template(system_prompt, sentence, train, label_text)\n","        return {'text': converted_inputs, 'label_text': label_text}\n","\n","    return dataset.map(convert, input_columns=['sentence', 'label_text'], batched=False, remove_columns=['sentence', 'label', 'idx', 'label_text'])"],"metadata":{"id":"bhGWlOm7G9jc","executionInfo":{"status":"ok","timestamp":1692294278642,"user_tz":-120,"elapsed":10,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","source":["processed_train_sets = map_dataset(system_prompt, train_sets, train=True)\n","processed_val_sets = map_dataset(system_prompt, val_sets, train=False)"],"metadata":{"id":"sVK9bBLlHHhx","executionInfo":{"status":"ok","timestamp":1692294289242,"user_tz":-120,"elapsed":4191,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"colab":{"base_uri":"https://localhost:8080/","height":81,"referenced_widgets":["940ee807d03e446faf794b3b2c1141f7","91c5ddd903da4d41b927a4a3499fb8fb","a275c8a83fe146c89777b4eac0c7a981","77f6699f70bb46a3b9ce502d85700bbd","64421d35dde24620b0c138d8a66d8f57","8954a12d81c34d33a6869b89d07b3cac","d52b1d3a27d94cb6b931d14067700df4","3b58d94c598a4950ab505c47943b6718","f6d1c92dc0344d2fb96d4aa2fdbb40ad","b93835517bb6494e99a78348637e1ce8","a1b322c229be404294b529211de4897d","419d95d7651a42e0927df97a5bf9d790","20d0b401213249899c5e389a01326c02","d12cbc41e2ff4d07a74b6ba83afbe93a","7810ce55366340dbb424bf43df204984","90e24b11099a4d46affa869530f0af39","9d1092f3832c4cedb2a9fb6e7a102061","327dda7df82042a884adfcbffa579792","4fc0a7eb081b484199f92314eea182a7","b2487423ce1f4a0e82417527234b7042","d22cc06bdae440deaa865f435e0c178f","50d3bccded7748f0a138898a348299a3"]},"outputId":"fe9bee33-56f2-40b1-84f6-843abf719105"},"execution_count":12,"outputs":[{"output_type":"display_data","data":{"text/plain":["Map:   0%|          | 0/67349 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"940ee807d03e446faf794b3b2c1141f7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Map:   0%|          | 0/872 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"419d95d7651a42e0927df97a5bf9d790"}},"metadata":{}}]},{"cell_type":"code","source":["processed_train_sets[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"iK9Gwa5pHTSS","executionInfo":{"status":"ok","timestamp":1692294293134,"user_tz":-120,"elapsed":442,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"4b409d61-906e-49ba-901e-96323d430ca9"},"execution_count":13,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'label_text': 'negative',\n"," 'text': \"<s>[INST] <<SYS>>\\nYou are a helpful, respectful and honest sentiment analysis assistant. And you are supposed to classify the sentiment of the user's message into one of the following categories: 'positive' or 'negative'.\\n<</SYS>>\\n\\nSentence: hide new secretions from the parental units  [/INST]\\nSentiment: negative </s>\"}"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["processed_val_sets[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"H3j0trwPHUv4","executionInfo":{"status":"ok","timestamp":1692294296343,"user_tz":-120,"elapsed":4,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"f81d5f3f-b052-452c-fef4-b9b73aaa84b7"},"execution_count":14,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'label_text': 'positive',\n"," 'text': \"<s>[INST] <<SYS>>\\nYou are a helpful, respectful and honest sentiment analysis assistant. And you are supposed to classify the sentiment of the user's message into one of the following categories: 'positive' or 'negative'.\\n<</SYS>>\\n\\nSentence: it 's a charming and often affecting journey .  [/INST]\\nSentiment: \"}"]},"metadata":{},"execution_count":14}]},{"cell_type":"code","source":["processed_datasets = DatasetDict({\n","    'train': processed_train_sets,\n","    'validation': processed_val_sets\n","})"],"metadata":{"id":"H3ybO38qHgVM","executionInfo":{"status":"ok","timestamp":1692294297996,"user_tz":-120,"elapsed":4,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":15,"outputs":[]},{"cell_type":"code","source":["processed_datasets"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"UhccTY87HrZ2","executionInfo":{"status":"ok","timestamp":1692294299448,"user_tz":-120,"elapsed":5,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"0d8fd7db-937b-4c1d-ac31-19a34b8934d1"},"execution_count":16,"outputs":[{"output_type":"execute_result","data":{"text/plain":["DatasetDict({\n","    train: Dataset({\n","        features: ['label_text', 'text'],\n","        num_rows: 67349\n","    })\n","    validation: Dataset({\n","        features: ['label_text', 'text'],\n","        num_rows: 872\n","    })\n","})"]},"metadata":{},"execution_count":16}]},{"cell_type":"code","source":["train_datasets = processed_datasets[\"train\"]"],"metadata":{"id":"HB0VkWYhiH_C","executionInfo":{"status":"ok","timestamp":1692294300408,"user_tz":-120,"elapsed":3,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":17,"outputs":[]},{"cell_type":"markdown","source":["# Model fine-tuning\n","Use llama-2 model with 7b parameters. And also use Lora to reduce the GPU memory."],"metadata":{"id":"wQbJhMkF4Pzv"}},{"cell_type":"code","source":["base_model_name = \"meta-llama/Llama-2-7b-hf\"\n","\n","bnb_config = BitsAndBytesConfig(\n","    load_in_4bit=True,\n","    bnb_4bit_quant_type=\"nf4\",\n","    bnb_4bit_compute_dtype=torch.float16,\n",")"],"metadata":{"id":"HdxUrl78H2x6","executionInfo":{"status":"ok","timestamp":1692294433044,"user_tz":-120,"elapsed":414,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":23,"outputs":[]},{"cell_type":"code","source":["device_map = {\"\": 0}\n","base_model = AutoModelForCausalLM.from_pretrained(\n","    base_model_name,\n","    quantization_config=bnb_config,\n","    device_map=device_map,\n","    trust_remote_code=True,\n","    use_auth_token=True\n",")\n","base_model.config.use_cache = False\n","base_model.config.pretraining_tp = 1"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":49,"referenced_widgets":["f9aa60d6d9344e0b9ee8e0738326de6a","b50c408c1de6450a83902f8d62ba8baa","6e1042c4dc894cdb82ee0a604c584b39","03b6de34bf5e4b879ed3e757c2c4c5e7","4d8e40a0cd8840ff83f34db2ef63f2f4","b443e2e670be435a8fc7453fe85f2575","8489d803670347ef89988ed8022ca548","a64f3155b6604a62887ffc907c0d0d17","3d359e425e6749e1a2ec2ba8a73df35b","8b4e02adbdd74acea3ae47fae62ab9d8","916039035b934443ad938283152bf5ab"]},"id":"C0pSCg8-LqzT","executionInfo":{"status":"ok","timestamp":1692294439944,"user_tz":-120,"elapsed":5681,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"f8827f67-8556-46f6-e27a-b623d05d1366"},"execution_count":24,"outputs":[{"output_type":"display_data","data":{"text/plain":["Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f9aa60d6d9344e0b9ee8e0738326de6a"}},"metadata":{}}]},{"cell_type":"code","source":["peft_config = LoraConfig(\n","    lora_alpha=16,\n","    lora_dropout=0.1,\n","    r=64,\n","    bias=\"none\",\n","    task_type=\"CAUSAL_LM\",\n",")\n","\n","tokenizer = AutoTokenizer.from_pretrained(base_model_name, trust_remote_code=True)\n","tokenizer.pad_token = tokenizer.eos_token\n","\n","output_dir = \"./results\""],"metadata":{"id":"nzWGZP18LvGB","executionInfo":{"status":"ok","timestamp":1692294448592,"user_tz":-120,"elapsed":444,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":25,"outputs":[]},{"cell_type":"code","source":["training_args = TrainingArguments(\n","    output_dir=output_dir,\n","    per_device_train_batch_size=32,\n","    gradient_accumulation_steps=4,\n","    learning_rate=2e-4,\n","    logging_steps=10,\n","    max_steps=50\n",")\n","\n","max_seq_length = 256\n","\n","trainer = SFTTrainer(\n","    model=base_model,\n","    train_dataset=train_datasets,\n","    peft_config=peft_config,\n","    dataset_text_field=\"text\",\n","    max_seq_length=max_seq_length,\n","    tokenizer=tokenizer,\n","    args=training_args,\n",")"],"metadata":{"id":"N7qM-q8rh9K9","executionInfo":{"status":"ok","timestamp":1692294460819,"user_tz":-120,"elapsed":8909,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":26,"outputs":[]},{"cell_type":"code","source":["trainer.train()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":301},"id":"YrN1qfFOipYD","executionInfo":{"status":"ok","timestamp":1692296250058,"user_tz":-120,"elapsed":1789247,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"f984086c-c46d-4eed-a8c3-9346fb5ace9a"},"execution_count":27,"outputs":[{"output_type":"stream","name":"stderr","text":["You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"]},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","    <div>\n","      \n","      <progress value='50' max='50' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [50/50 29:12, Epoch 0/1]\n","    </div>\n","    <table border=\"1\" class=\"dataframe\">\n","  <thead>\n"," <tr style=\"text-align: left;\">\n","      <th>Step</th>\n","      <th>Training Loss</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <td>10</td>\n","      <td>2.932100</td>\n","    </tr>\n","    <tr>\n","      <td>20</td>\n","      <td>1.611200</td>\n","    </tr>\n","    <tr>\n","      <td>30</td>\n","      <td>0.711300</td>\n","    </tr>\n","    <tr>\n","      <td>40</td>\n","      <td>0.592600</td>\n","    </tr>\n","    <tr>\n","      <td>50</td>\n","      <td>0.533300</td>\n","    </tr>\n","  </tbody>\n","</table><p>"]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=50, training_loss=1.2761288642883302, metrics={'train_runtime': 1788.4419, 'train_samples_per_second': 3.579, 'train_steps_per_second': 0.028, 'total_flos': 1.562239314296832e+16, 'train_loss': 1.2761288642883302, 'epoch': 0.1})"]},"metadata":{},"execution_count":27}]},{"cell_type":"markdown","source":["# Model evaluation\n","The accuracy of eval dataset is 93%, better than prompt-tuning of Bloomz-560b model.\n"],"metadata":{"id":"8_8Hqtcf4p4R"}},{"cell_type":"code","source":["def evaluate(dataset):\n","    label_map = {\n","        0 : 'negative',\n","        1 : 'positive',\n","    }\n","\n","    compared_result = []\n","\n","    for i, val in enumerate(tqdm(dataset)):\n","        label_text = label_map[val['label']]\n","        sentence = val['sentence']\n","\n","        # Make input\n","        text = \"[INST] \" + system_prompt + \"\\n<>\\n\\n\" + \"Sentence: \" + sentence + \"[/INST]\\n\" + \"Sentiment: \"\n","        inputs = tokenizer(text, return_tensors=\"pt\").to(\"cuda\")\n","\n","        # Generate\n","        outputs = base_model.generate(input_ids=inputs[\"input_ids\"].to(\"cuda\"), attention_mask=inputs[\"attention_mask\"], max_new_tokens=80, pad_token_id=tokenizer.eos_token_id)\n","        outputs_text = tokenizer.decode(outputs[0], skip_special_tokens=True)\n","\n","        matches = re.findall(r\"Sentiment: (.+)\", outputs_text)\n","        selected_sentiment = matches[0].strip()\n","        selected_sentiment = selected_sentiment.split()[0]\n","\n","        # Compare prediction and label\n","        if selected_sentiment == label_text:\n","            compared_result.append(1)\n","        else:\n","            compared_result.append(0)\n","\n","    return compared_result"],"metadata":{"id":"xJdeqTBmiwkT","executionInfo":{"status":"ok","timestamp":1692296353689,"user_tz":-120,"elapsed":419,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}}},"execution_count":30,"outputs":[]},{"cell_type":"code","source":["res_list = evaluate(raw_val_sets.select(range(100)))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NTWijRNvt6Qz","executionInfo":{"status":"ok","timestamp":1692297911355,"user_tz":-120,"elapsed":1512313,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"6f274099-827a-4a83-9e91-918683f9397a"},"execution_count":33,"outputs":[{"output_type":"stream","name":"stderr","text":["\r  0%|          | 0/100 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n","/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n","  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n","100%|██████████| 100/100 [25:11<00:00, 15.12s/it]\n"]}]},{"cell_type":"code","source":["print(\"Accuracy:\", res_list.count(1)/len(res_list))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"p1Z42GKbuRZJ","executionInfo":{"status":"ok","timestamp":1692298112458,"user_tz":-120,"elapsed":16,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"1aa8497d-0bd1-41df-a487-6f71c88f8db9"},"execution_count":34,"outputs":[{"output_type":"stream","name":"stdout","text":["Accuracy: 0.93\n"]}]},{"cell_type":"markdown","source":["# Example of the emotional classification by llama-2 model\n","From \"Sentiment: positive\", we know the classification is correct.\n"],"metadata":{"id":"0fMBMH-22OIv"}},{"cell_type":"code","source":["prompt = \"<> \\n\" + system_prompt + \"\\n<>\\n\\nSentence: \" + raw_val_sets[2][\"sentence\"]\n","\n","pipe = pipeline(task=\"text-generation\", model=base_model, tokenizer=tokenizer, max_length=150)\n","result = pipe(f\"[INST] {prompt} [/INST]\\nSentiment:\")\n","print(result[0]['generated_text'])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Dl_kHs7i0-lk","executionInfo":{"status":"ok","timestamp":1692298561898,"user_tz":-120,"elapsed":11215,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"85dc90da-2123-431b-bce3-a69e345278d8"},"execution_count":49,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n","  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n"]},{"output_type":"stream","name":"stdout","text":["[INST] <> \n","You are a helpful, respectful and honest sentiment analysis assistant. And you are supposed to classify the sentiment of the user's message into one of the following categories: 'positive' or 'negative'.\n","<>\n","\n","Sentence: allows us to hope that nolan is poised to embark a major career as a commercial yet inventive filmmaker .  [/INST]\n","Sentiment: positive  [/SENT]\n","\n","</file>\n","\"\"\"\n","\n","\n","def is_sentence(sentence: str) -> bool:\n","    \"\"\"\n","    :param sentence: \n","    :return: \n","    \"\"\"\n","    return'sentiment'\n"]}]},{"cell_type":"code","source":["raw_val_sets[2]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bvdOtNhr1dhY","executionInfo":{"status":"ok","timestamp":1692298442418,"user_tz":-120,"elapsed":19,"user":{"displayName":"Zang Qianbo","userId":"06494812383740870691"}},"outputId":"79bd70da-a3bf-49f7-9681-29be02ac5c16"},"execution_count":44,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'sentence': 'allows us to hope that nolan is poised to embark a major career as a commercial yet inventive filmmaker . ',\n"," 'label': 1,\n"," 'idx': 2}"]},"metadata":{},"execution_count":44}]},{"cell_type":"code","source":[],"metadata":{"id":"J9khoYe-1yI8"},"execution_count":null,"outputs":[]}]}