playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('data/sentiment_comp_qaie_pairs.pkl','rb') as f:\n",
    "    pairs = pickle.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12325\n",
      "12272\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['智通财经APP讯，太平洋网络(00543)发布公告，将于2023年6月12日派发截至2022年12月31日止年度的末期股息每股0.1元人民币。\\n---\\n请从上文中抽取出所有公司/机构、对应的在本文中的情感倾向（积极、消极、中性）以及原因。\\n并用这样的格式返回：\\n{\"ORG\":..., \"sentiment\":..., \"reason\":...}',\n",
       " '{\"ORG\": \"太平洋网络\", \"sentiment\": \"积极\", \"reason\": \"宣布派发股息每股0.1元人民币\"}']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(pairs))\n",
    "pairs = [p for p in pairs if p[1] not in ['无','']]\n",
    "print(len(pairs))\n",
    "\n",
    "pairs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('data/sentiment_comp_ie.json','w',encoding='utf8') as f:\n",
    "    for p in pairs:\n",
    "        line = {\"q\":p[0],\"a\":p[1]}\n",
    "        f.write(json.dumps(line,ensure_ascii=False))\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 打乱顺序重新排列\n",
    "import json\n",
    "import random\n",
    "with open('data/sentiment_comp_ie_shuffled.json','w',encoding='utf8') as f:\n",
    "    random_pairs = random.sample(pairs,k=len(pairs))\n",
    "    for p in random_pairs:\n",
    "        line = {\"q\":p[0],\"a\":p[1]}\n",
    "        f.write(json.dumps(line,ensure_ascii=False))\n",
    "        f.write('\\n')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## baichuan-7B inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/gby/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "# 下载百川大模型看看\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"baichuan-inc/baichuan-7B\", trust_remote_code=True)\n",
    "model = AutoModelForCausalLM.from_pretrained(\"baichuan-inc/baichuan-7B\", device_map=\"auto\", trust_remote_code=True).half()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 尝试从大模型中获取情感标签的概率值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"问：今天心情还真不错。这句话的情感是怎样的？答：这句话的情感是：\"\n",
    "inputs = tokenizer.encode(text, return_tensors='pt').to('cuda:0')\n",
    "# outputs = model.generate(inputs,max_new_tokens=50,do_sample=True)\n",
    "# output = outputs[0][len(inputs[0]):]\n",
    "# tokenizer.decode(output,skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ['消极', '负面', '难过'] score(prob*100) = 0.2\n",
      "1 ['积极', '正面', '开心'] score(prob*100) = 0.07\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "text = \"ChatGPT的推出对百度的搜索业务产生的强烈冲击，搜索引擎的作用性在降低\"\n",
    "\n",
    "prompt = f\"问：{text}这句话的情感是怎样的？答：这句话的情感是：\"\n",
    "inputs = tokenizer.encode(prompt, return_tensors='pt').to('cuda:0')\n",
    "\n",
    "last_logits = model(inputs).logits[:,-1,:]\n",
    "probs = torch.softmax(last_logits, dim=-1)\n",
    "\n",
    "targets = {\n",
    "    0: ['消极','负面','难过'],\n",
    "    1: ['积极','正面','开心']\n",
    "}\n",
    "for key in targets:\n",
    "    target_probs = []\n",
    "    for w in targets[key]:\n",
    "        # 取出非 bos 的第一个token（似乎无法通过 add_special_tokens=False 去掉）\n",
    "        idx = tokenizer.encode(w)[1]\n",
    "        current_prob = probs[0, idx].item()\n",
    "        # print(w, current_prob)\n",
    "        target_probs.append(current_prob)\n",
    "    print(key,targets[key],'score(prob*100) =',round(sum(target_probs)*100/len(target_probs),2))\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'不耐烦，极其不耐烦。为什么会不耐烦呢？因为我们总是觉得对方不了解情况。1 先了解一下事情的始末吧！在说这句话的时候，对方可能会说：“事情不是这样子的呀'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs = model.generate(inputs,max_new_tokens=50,do_sample=True)\n",
    "output = outputs[0][len(inputs[0]):]\n",
    "tokenizer.decode(output,skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaiChuanForCausalLM(\n",
       "  (model): Model(\n",
       "    (embed_tokens): Embedding(64000, 4096, padding_idx=0)\n",
       "    (layers): ModuleList(\n",
       "      (0-31): 32 x DecoderLayer(\n",
       "        (self_attn): Attention(\n",
       "          (W_pack): Linear(in_features=4096, out_features=12288, bias=False)\n",
       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (rotary_emb): RotaryEmbedding()\n",
       "        )\n",
       "        (mlp): MLP(\n",
       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
       "          (act_fn): SiLUActivation()\n",
       "        )\n",
       "        (input_layernorm): RMSNorm()\n",
       "        (post_attention_layernorm): RMSNorm()\n",
       "      )\n",
       "    )\n",
       "    (norm): RMSNorm()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=4096, out_features=64000, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import PeftModel\n",
    "model = PeftModel.from_pretrained(model, \"weights/sentiment_comp_ie_shuffled_baichuan-7B\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"\"\"ChatGPT的提出对谷嘎、万度的搜索业务产生巨大打击，传统搜索引擎的作用性降低了。\n",
    "与此同时，OChat，Linguo等新兴语义搜索公司，迅速推出自己的类ChatGPT模型，并结合进自家搜索引擎，受到了很多用户的青睐。\n",
    "腾势、艾里等公司表示会迅速跟进ChatGPT和AIGC的发展，并预计在年底前推出自己的大模型。\n",
    "大型图片供应商视觉中国称ChatGPT对公司业务暂无影响，还在观望状态。\n",
    "（本文图片来自视觉中国，上观新闻为您报道。）\n",
    "更多报道：\n",
    "- 亚牛逊公司关于AIGC的表态\n",
    "- 巨硬公司昨日在A股上市\n",
    "---\n",
    "请从上文中抽取出所有公司，以及对应的在本文中的情感倾向（积极、消极、中性）以及原因。\n",
    "请用这样的格式返回：\n",
    "{\"ORG\":..., \"sentiment\":..., \"reason\":...}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1. {\"ORG\":\"百度\",\"SENTIMENT\":\"+20%\", \"REASON\":\"新产品发布/更新;技术突破;行业领先地位提升;市场份额扩大;营收增长;股价上涨\"; }\n",
      "2. {\"ORG\":\"阿里巴巴集团\",\"SENTIMENT\":\"-50%\", \"REASON\":\"竞争对手发展迅猛;业绩下滑;市值缩水;股价下跌\"; }\n",
      "3. {\"ORG\":\"腾讯控股有限公司\",\"SENTIMENT\":\"+40%\", \"REASON\":\"新产品发布\n"
     ]
    }
   ],
   "source": [
    "from transformers import TextStreamer\n",
    "streamer = TextStreamer(tokenizer,skip_prompt=True,skip_special_tokens=True)\n",
    "\n",
    "inputs = tokenizer(text, return_tensors='pt')\n",
    "inputs = inputs.to('cuda:0')\n",
    "output = model.generate(**inputs, max_new_tokens=128,repetition_penalty=1.1, streamer=streamer)\n",
    "# 模型非常自信！（类似于模型自动纠错的能力）同时幻觉十分严重，哈哈哈（当然，这只是训练了200steps）"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ChatGLM inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 8/8 [00:12<00:00,  1.52s/it]\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModel,AutoTokenizer\n",
    "device=torch.device(2)\n",
    "\n",
    "model_path = \"THUDM/chatglm-6b\"\n",
    "# model_glm = AutoModel.from_pretrained(model_path, trust_remote_code=True).half().to(device)\n",
    "model_glm = AutoModel.from_pretrained(model_path, trust_remote_code=True,device_map='auto')\n",
    "tokenizer_glm = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)\n",
    "# model_glm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load LoRA\n",
    "model_glm = PeftModel.from_pretrained(model_glm, \"weights/sentiment_comp_ie\").half()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "我不确定您所指的“艾里巴巴公司”是指哪个公司。如果您能提供更多上下文或信息,我将尽力回答您的问题。\n"
     ]
    }
   ],
   "source": [
    "from transformers import TextStreamer\n",
    "streamer = TextStreamer(tokenizer_glm,skip_prompt=True,skip_special_tokens=True)\n",
    "\n",
    "inputs = tokenizer_glm(\"我们说的艾里巴巴公司，指的是\", return_tensors='pt')\n",
    "inputs = inputs.to('cuda:2')\n",
    "output = model_glm.generate(**inputs, max_new_tokens=1024,repetition_penalty=1.1, streamer=streamer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "融资资金将用于完善生成式AI新引擎的构建。\n",
      "\n",
      "创业邦获悉，自然语言技术AI服务商竹间智能宣布已完成D2轮融资，由金浦投资、金库资本、江苏文投、隽赐资本等联合投资。至今，竹间智能已累计完成7轮融资，陆续引入科沃斯、云晖资本、中银国际等股东机构。值得一提的是，竹间智能已经正式推出运用类ChatGPT技术的成熟AIGC产品，并即将开启新一轮融资。\n",
      "\n",
      "融资资金将用于完善生成式AI新引擎的构建，把以ChatGPT为代表的大语言模型和AIGC技术全面融入竹间产品体系，结合大小模型构建双引擎驱动产品迭代和技术升级，平衡大小模型的优缺点，全面焕新产品功能和服务模式，并正式将产品同步推向海外市场，成为服务全球企业和用户的跨国NLP能力厂商。\n",
      "\n",
      "竹间智能由前微软（亚洲）互联网工程院副院长简仁贤于2015年创办，致力于以自然语言处理、情感计算、深度学习、知识工程、文本处理等人工智能技术为基础，将AI能力惠及千行百业。\n",
      "\n",
      "---\n",
      "请从上文中抽取出所有公司，以及对应的在本文中的情感倾向（积极、消极、中性）以及原因。\n",
      "请用这样的格式返回：\n",
      "{\"ORG\":..., \"sentiment\":..., \"reason\":...}\n"
     ]
    }
   ],
   "source": [
    "print(text)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tokenizer自动把中文括号转成英文括号了...所以你直接在输出里面可能无法直接匹配到 prompt"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ChatGLM2-6B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModel\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"THUDM/chatglm2-6b\", trust_remote_code=True)\n",
    "# model = AutoModel.from_pretrained(\"THUDM/chatglm2-6b\", trust_remote_code=True, device='cuda:0')\n",
    "model = AutoModel.from_pretrained(\"THUDM/chatglm2-6b\", trust_remote_code=True, device_map='auto')\n",
    "# model = model.eval()\n",
    "# model\n",
    "# response, history = model.chat(tokenizer, \"你好\", history=[])\n",
    "# print(response)\n",
    "\n",
    "# response, history = model.chat(tokenizer, \"晚上睡不着应该怎么办\", history=history)\n",
    "# print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.hf_device_map\n",
    "model.hf_device_map['transformer.output_layer'] = model.hf_device_map['transformer.embedding']\n",
    "model = AutoModel.from_pretrained(\"THUDM/chatglm2-6b\", trust_remote_code=True, device_map=model.hf_device_map)\n",
    "model.hf_device_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = tokenizer('你好',return_tensors='pt')\n",
    "res_glm = tokenizer_glm('你好',return_tensors='pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = res['input_ids'].clone()\n",
    "model(input_ids=res['input_ids'],labels=labels)\n",
    "\n",
    "# model_glm = model_glm.half()\n",
    "# labels_glm = res_glm['input_ids'].clone()\n",
    "# model_glm(input_ids=res_glm['input_ids'],labels=labels_glm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**对比一下 ChatGLM 跟 ChatGLM2 的结构差别：**\n",
    "\n",
    "ChatGLM:\n",
    "```python\n",
    "ChatGLMForConditionalGeneration(\n",
    "  (transformer): ChatGLMModel(\n",
    "    (word_embeddings): Embedding(130528, 4096)\n",
    "    (layers): ModuleList(\n",
    "      (0-27): 28 x GLMBlock(\n",
    "        (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
    "        (attention): SelfAttention(\n",
    "          (rotary_emb): RotaryEmbedding()\n",
    "          (query_key_value): Linear(in_features=4096, out_features=12288, bias=True)\n",
    "          (dense): Linear(in_features=4096, out_features=4096, bias=True)\n",
    "        )\n",
    "        (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
    "        (mlp): GLU(\n",
    "          (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n",
    "          (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n",
    "        )\n",
    "      )\n",
    "    )\n",
    "    (final_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
    "  )\n",
    "  (lm_head): Linear(in_features=4096, out_features=130528, bias=False)\n",
    "```\n",
    "\n",
    "ChatGLM2:\n",
    "```python\n",
    "ChatGLMForConditionalGeneration(\n",
    "  (transformer): ChatGLMModel(\n",
    "    (embedding): Embedding(\n",
    "      (word_embeddings): Embedding(65024, 4096)   # <-- smaller vocab size\n",
    "    )\n",
    "    (rotary_pos_emb): RotaryEmbedding()\n",
    "    (encoder): GLMTransformer(\n",
    "      (layers): ModuleList(\n",
    "        (0-27): 28 x GLMBlock(\n",
    "          (input_layernorm): RMSNorm()   # <-- LayerNorm to RMSNorm\n",
    "          (self_attention): SelfAttention(\n",
    "            (query_key_value): Linear(in_features=4096, out_features=4608, bias=True)   # <-- smaller attention out_features\n",
    "            (core_attention): CoreAttention(\n",
    "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
    "            )\n",
    "            (dense): Linear(in_features=4096, out_features=4096, bias=False)\n",
    "          )\n",
    "          (post_attention_layernorm): RMSNorm()\n",
    "          (mlp): MLP(                # <-- GLU to MLP\n",
    "            (dense_h_to_4h): Linear(in_features=4096, out_features=27392, bias=False)\n",
    "            (dense_4h_to_h): Linear(in_features=13696, out_features=4096, bias=False)\n",
    "          )\n",
    "        )\n",
    "      )\n",
    "      (final_layernorm): RMSNorm()\n",
    "    )\n",
    "    (output_layer): Linear(in_features=4096, out_features=65024, bias=False)    # <-- smaller out_features\n",
    "  )\n",
    ")\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=4096, out_features=130528, bias=False)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_glm.lm_head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=4096, out_features=65024, bias=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.transformer.output_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"ORG\": \"ChatGPT\", \"sentiment\": \"消极\", \"reason\": \"ChatGPT的提出对谷嘎、万度的搜索业务产生巨大打击，传统搜索引擎的作用性降低了。\"},\n",
      "{\"ORG\": \"OChat\", \"sentiment\": \"积极\", \"reason\": \"OChat，Linguo等新兴语义搜索公司，迅速推出自己的类ChatGPT模型，并结合进自家搜索引擎，受到了很多用户的青睐。\"},\n",
      "{\"ORG\": \"腾势\", \"sentiment\": \"积极\", \"reason\": \"腾势、艾里等公司表示会迅速跟进ChatGPT和AIGC的发展，并预计在年底前推出自己的大模型。\"},\n",
      "{\"ORG\": \"视觉中国\", \"sentiment\": \"中性\", \"reason\": \"大型图片供应商视觉中国称ChatGPT对公司业务暂无影响，还在观望状态。\"},\n",
      "{\"ORG\": \"亚牛逊公司\", \"sentiment\": \"中性\", \"reason\": \"亚牛逊公司关于AIGC的表态中并未提及ChatGPT对公司业务的影响。\"},\n",
      "{\"ORG\": \"巨硬公司\", \"sentiment\": \"中性\", \"reason\": \"巨硬公司昨日在A股上市与ChatGPT对公司业务的影响无关。\"}\n"
     ]
    }
   ],
   "source": [
    "print(model.chat(tokenizer,query=text)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import PeftModel\n",
    "model = PeftModel.from_pretrained(model, \"weights/sentiment_comp_ie_chatglm2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"ORG\": \"谷嘎\", \"sentiment\": \"消极\", \"reason\": \"ChatGPT的提出对其搜索业务产生巨大打击\"}\n",
      "{\"ORG\": \"万度\", \"sentiment\": \"消极\", \"reason\": \"传统搜索引擎的作用性降低了\"}\n",
      "{\"ORG\": \"OChat\", \"sentiment\": \"积极\", \"reason\": \"迅速推出自己的类ChatGPT模型，并结合进自家搜索引擎，受到了很多用户的青睐\"}\n",
      "{\"ORG\": \"Linguo\", \"sentiment\": \"积极\", \"reason\": \"迅速推出自己的类ChatGPT模型，并结合进自家搜索引擎，受到了很多用户的青睐\"}\n",
      "{\"ORG\": \"腾势\", \"sentiment\": \"积极\", \"reason\": \"会迅速跟进ChatGPT和AIGC的发展，并预计在年底前推出自己的大模型\"}\n",
      "{\"ORG\": \"艾里\", \"sentiment\": \"积极\", \"reason\": \"会迅速跟进ChatGPT和AIGC的发展，并预计在年底前推出自己的大模型\"}\n",
      "{\"ORG\": \"视觉中国\", \"sentiment\": \"中性\", \"reason\": \"称ChatGPT对公司业务暂无影响，还在观望状态\"}\n"
     ]
    }
   ],
   "source": [
    "print(model.chat(tokenizer,text)[0])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HC3 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "from datasets import load_from_disk\n",
    "\n",
    "hc3 = load_from_disk('data/hc3_chatgpt_qa_all')\n",
    "hc3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': '盗贼天赋盗贼怎么加天赋?知道告诉一下下啦~~ ',\n",
       " 'answer': '如果你在玩角色扮演游戏（RPG），那么你可能是在问如何在游戏中给你的盗贼角色加天赋。具体方法可能因游戏而异，但通常有以下几种方法： \\n1. 在游戏开始时选择盗贼天赋：在游戏开始时，你可以选择你想要的天赋。这通常是通过选择不同的角色种族或职业来实现的。 \\n2. 在游戏进程中获得天赋：在游戏进程中，你可能会获得一些与盗贼相关的天赋。这可能是通过完成任务、升级或解锁新的技能来实现的。 \\n3. 使用道具或装备获得天赋：你可能会发现一些道具或装备，它们可以赋予你一些盗贼天赋。这些道具或装备通常是随机生成的，或者是你在游戏进程中获得的奖励。 \\n希望这些信息对你有所帮助！'}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hc3[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('data/hc3_chatgpt_zh_specific_qa.json','w',encoding='utf8') as f:\n",
    "    for i in range(len(hc3)):\n",
    "        # line = {'q':hc3[i]['question'], 'a':hc3[i]['answer']}\n",
    "        line = {'q':'问：'+hc3[i]['question'], 'a':'答：'+hc3[i]['answer']}\n",
    "        line = json.dumps(line, ensure_ascii=False)\n",
    "        f.write(line)\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from peft import PeftModel\n",
    "from transformers import TextStreamer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"baichuan-inc/baichuan-7B\", trust_remote_code=True)\n",
    "import torch\n",
    "device = torch.device('cuda:2')\n",
    "# model = AutoModelForCausalLM.from_pretrained(\"baichuan-inc/baichuan-7B\", trust_remote_code=True).to(device)\n",
    "model = AutoModelForCausalLM.from_pretrained(\"baichuan-inc/baichuan-7B\", device_map=\"auto\", trust_remote_code=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chat(text):\n",
    "    streamer = TextStreamer(tokenizer,skip_prompt=True,skip_special_tokens=True)\n",
    "\n",
    "    inputs = tokenizer(\"问：\"+text+\"答：\", return_tensors='pt')\n",
    "    inputs = inputs.to('cuda:2')\n",
    "    output = model.generate(**inputs, max_new_tokens=128,\n",
    "                            do_sample=True,\n",
    "                            repetition_penalty=1.1, \n",
    "                            begin_suppress_tokens=[tokenizer.eos_token_id],\n",
    "                            # streamer=streamer\n",
    "                            )\n",
    "    output = output[0][inputs.input_ids.shape[1]:] # 这样可以防止输出prompt部分\n",
    "    return tokenizer.decode(output,skip_special_tokens=True)\n",
    "    # return output\n",
    "    \n",
    "# chat('你怎么把断了的螺丝钉钻出来？')\n",
    "# tokenizer.eos_token_id\n",
    "# tokenizer.encode('  ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = PeftModel.from_pretrained(model, \"weights/rulai_plus_baichuan-7B/\")\n",
    "model = PeftModel.from_pretrained(model, \"RLHF/weights/baichaun_rlhf_beyond_chinese_test_6step_40\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "有事情想找老公商量又不好意思？\n"
     ]
    }
   ],
   "source": [
    "output = chat('哎，今天好烦啊')\n",
    "print(output)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "len = 2500  ~ 9G\n",
    "len = 5000  ~ 40G"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# InternLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n",
    "from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer\n",
    "from peft import PeftModel, get_peft_model\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained('internlm/internlm-7b', trust_remote_code=True, device_map=\"auto\") \n",
    "tokenizer = AutoTokenizer.from_pretrained('internlm/internlm-7b', trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "InternLMForCausalLM(\n",
       "  (model): InternLMModel(\n",
       "    (embed_tokens): Embedding(103168, 4096, padding_idx=0)\n",
       "    (layers): ModuleList(\n",
       "      (0-31): 32 x InternLMDecoderLayer(\n",
       "        (self_attn): InternLMAttention(\n",
       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "          (k_proj): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "          (v_proj): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "          (rotary_emb): InternLMRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): InternLMMLP(\n",
       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
       "          (act_fn): SiLUActivation()\n",
       "        )\n",
       "        (input_layernorm): InternLMRMSNorm()\n",
       "        (post_attention_layernorm): InternLMRMSNorm()\n",
       "      )\n",
       "    )\n",
       "    (norm): InternLMRMSNorm()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=4096, out_features=103168, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 按照惯例，先观察一下模型结构\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"正向\":1,\"负向\":0,\"未提及\":0,\"中\":0,\"推荐程度\":3,\"性价比\":3,\"折扣力度\":0,\"装修情况\":3,\"嘈杂情况\":0,\"就餐空间\":3,\"卫生情况\":3,\"分量\":3,\"口感\":3,\"外观\":3,\"推荐程度\":3,\"再次消费的意愿\":3}<eoa>\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from transformers import TextStreamer\n",
    "def predict(text,instruction):\n",
    "    streamer = TextStreamer(tokenizer,skip_prompt=True,skip_special_tokens=True)\n",
    "    prompt = instruction + text + '\\n输出：'\n",
    "    inputs = tokenizer(prompt, return_tensors='pt')\n",
    "    inputs = inputs.to('cuda:0')\n",
    "    output = model.generate(**inputs, max_new_tokens=1024,\n",
    "                            # do_sample=True,\n",
    "                            # repetition_penalty=1.1, \n",
    "                            temperature=0.8,\n",
    "                            top_p=0.8,\n",
    "                            eos_token_id=(2, 103028),\n",
    "                            # begin_suppress_tokens=[tokenizer.eos_token_id],\n",
    "                            streamer=streamer\n",
    "                            )\n",
    "    output = output[0][inputs.input_ids.shape[1]:] # 这样可以防止输出prompt部分\n",
    "    return tokenizer.decode(output,skip_special_tokens=True)\n",
    "\n",
    "text = \"首先是环境很舒服，我去过附近不少咖啡馆，像回声，咖啡王子一号店，星巴克等等，感觉他们家的环境绝对是顶尖的之一。\\\n",
    "星巴克适合商务人士喝杯咖啡小憩一会，回声适合安安静静的喝咖啡读书度过一个下午。这里怎么说呢，介于两者之间吧，有咖啡有西餐，\\\n",
    "    在这晃一天都没问题。对于我这单纯学习狗来说，音乐声音有点大，另外每个桌子都有电源，但是mac电脑的电源很不好插。\\\n",
    "        点了一杯香草拿铁，觉得有点甜，不过有免费的柠檬水，可以解渴。图中是芝士玉米圆饼，芝士用料很足，个人感觉不错。\\\n",
    "            有时间的话，在这里喝喝咖啡，读读书度过一个下午真是太舒服了。\"\n",
    "\n",
    "instruction = \"给定一段文字，请你进行细粒度方面情感分析，具体包括以下这些方面：交通是否便利,距离商圈远近,是否容易寻找,排队等候时间,服务人员态度,是否容易停车,点菜/上菜速度,价格水平,性价比,折扣力度,装修情况,嘈杂情况,就餐空间,卫生情况,分量,口感,外观,推荐程度,本次消费感受,再次消费的意愿。请抽取出所有方面的情感倾向：正向:1, 中性:0, 负向:-1, 未提及:-2，用json格式返回结果。 评论如下：\"\n",
    "output = predict(text, instruction)\n",
    "\n",
    "# json.loads(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"交通是否便利\": \"未提及\", \"距离商圈远近\": \"未提及\", \"是否容易寻找\": \"未提及\", \"排队等候时间\": \"未提及\", \"服务人员态度\": \"未提及\", \"是否容易停车\": \"未提及\", \"点菜/上菜速度\": \"未提及\", \"价格水平\": \"未提及\", \"性价比\": \"未提及\", \"折扣力度\": \"未提及\", \"装修情况\": \"正面\", \"嘈杂情况\": \"中性\", \"就餐空间\": \"正面\", \"卫生情况\": \"正面\", \"分量\": \"未提及\", \"口感\": \"中性\", \"外观\": \"未提及\", \"推荐程度\": \"未提及\", \"本次消费感受\": \"正面\", \"再次消费的意愿\": \"正面\"}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'交通是否便利': '未提及',\n",
       " '距离商圈远近': '未提及',\n",
       " '是否容易寻找': '未提及',\n",
       " '排队等候时间': '未提及',\n",
       " '服务人员态度': '未提及',\n",
       " '是否容易停车': '未提及',\n",
       " '点菜/上菜速度': '未提及',\n",
       " '价格水平': '未提及',\n",
       " '性价比': '未提及',\n",
       " '折扣力度': '未提及',\n",
       " '装修情况': '正面',\n",
       " '嘈杂情况': '中性',\n",
       " '就餐空间': '正面',\n",
       " '卫生情况': '正面',\n",
       " '分量': '未提及',\n",
       " '口感': '中性',\n",
       " '外观': '未提及',\n",
       " '推荐程度': '未提及',\n",
       " '本次消费感受': '正面',\n",
       " '再次消费的意愿': '正面'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lora_path = '../XiaoWen/LLM-Tuning-master/weights/aspect_sentiment_jsonfy_output_10k_plus-internlm-chat-7b'\n",
    "model = PeftModel.from_pretrained(model, lora_path)\n",
    "\n",
    "\n",
    "json.loads(predict(text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# !!! 重要经验\n",
    "generate参数的影响巨大，应该跟 modeling_xxx.py 中默认使用的参数对应，能保证比较好的效果！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"交通是否便利\": \"未提及\", \"距离商圈远近\": \"未提及\", \"是否容易寻找\": \"未提及\", \"排队等候时间\": \"未提及\", \"服务人员态度\": \"未提及\", \"是否容易停车\": \"未提及\", \"点菜/上菜速度\": \"正面\", \"价格水平\": \"未提及\", \"性价比\": \"未提及\", \"折扣力度\": \"未提及\", \"装修情况\": \"未提及\", \"嘈杂情况\": \"未提及\", \"就餐空间\": \"未提及\", \"卫生情况\": \"未提及\", \"分量\": \"未提及\", \"口感\": \"正面\", \"外观\": \"未提及\", \"推荐程度\": \"未提及\", \"本次消费感受\": \"正面\", \"再次消费的意愿\": \"未提及\"}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'点菜/上菜速度': '正面', '口感': '正面', '本次消费感受': '正面'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"很喜欢这家的餐，味道正宗，口感很好，是我喜欢的味道，满意值得推荐包装仔细，\\\n",
    "    味道很不错，小哥送的也很快，点赞！味道很惊喜，口味很赞👍相当的满足！\"\n",
    "output_d = json.loads(predict(text))\n",
    "\n",
    "output_d = {k:v for k,v in output_d.items() if v != '未提及'}\n",
    "output_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "答案：{\"pos\":0,\"neg\":1,\"neu\":0,\"text\": \"喝过最难喝的奶茶，一股水的味道，根本就喝不到奶茶味，全倒了，浪费。芋泥就一点点，珍珠就6小颗。服气了！差评，差评。\"}<eoa>\n",
      "\n",
      "答案：{\"pos\":0,\"neg\":1,\"neu\":0,\"text\": \"喝过最难喝的奶茶，一股水的味道，根本就喝不到奶茶味，全倒了，浪费。芋泥就一点点，珍珠就6小颗。服气了！差评，差评。\"}<eoa>\n"
     ]
    }
   ],
   "source": [
    "with model.disable_adapter(): # 禁用 lora\n",
    "    output = predict(text)\n",
    "    try:\n",
    "        output_d = json.loads(output)\n",
    "        output_d = {k:v for k,v in output_d.items() if v != '未提及'}\n",
    "        print(output_d)\n",
    "    except:\n",
    "        print(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## baichuan2:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2,3\"\n",
    "from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer\n",
    "from peft import PeftModel, get_peft_model\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained('../DCAI-share/llm/Baichuan2-7B-Base', trust_remote_code=True, device_map=\"auto\").half()\n",
    "tokenizer = AutoTokenizer.from_pretrained('../DCAI-share/llm/Baichuan2-7B-Base', trust_remote_code=True)\n",
    "# model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import PeftModel\n",
    "# lora_path = 'weights/absa_instruction-first_d1_train-1000-Baichuan2-7B-Base'\n",
    "# model = PeftModel.from_pretrained(model, lora_path)\n",
    "\n",
    "lora_path3 = 'weights/absa_lines-output_d1_train-1000-Baichuan2-7B-Base'\n",
    "model.load_adapter(lora_path3, adapter_name='lines-output_d1')\n",
    "model.set_adapter('lines-output_d1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "食品评价：正面\n",
      "饮品评价：正面\n",
      "价格水平：正面\n",
      "卫生情况：正面\n",
      "服务人员态度：正面\n",
      "停车方便程度：未提及\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'食品评价：正面\\n饮品评价：正面\\n价格水平：正面\\n卫生情况：正面\\n服务人员态度：正面\\n停车方便程度：未提及'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import TextStreamer\n",
    "streamer = TextStreamer(tokenizer,skip_prompt=True,skip_special_tokens=True)\n",
    "\n",
    "text = \"\"\"我们又来了，自从东方广场这间分店开张以来，我们五个人已经是第三次消费，有团购这么划算的优惠，就多来一些，反正到哪里都是吃的啦！人气比刚开始时回落一点，正常现象。服务都是一如既往的好，用团购券买单还可以享用抽奖中的沙律，满意。总结之前两次得出的经验，本人认为鸡皇牛扒比杂扒好吃太多，价钱好像差不多吧。鸡扒拌天使面，可能鸡扒太肥，感觉一般。照烧鱿鱼筒是每次都点的，值得推介。长豆角也很野味，不错。这次好抽中两份奖品，要准备好下一次的钱了，再优惠些吧！\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\\\"方面1：情感类别\\n方面2：情感类别\\n...\\\"\\n输出：\"\"\"\n",
    "inputs = tokenizer(text, return_tensors='pt')\n",
    "inputs = inputs.to('cuda:0')\n",
    "output = model.generate(**inputs, max_new_tokens=128,repetition_penalty=1.1, streamer=streamer)\n",
    "output = output[0][inputs.input_ids.shape[1]:] # 这样可以防止输出prompt部分\n",
    "tokenizer.decode(output,skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chinese-LLaMa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/gby/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda-11.8/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 118\n",
      "CUDA SETUP: Loading binary /root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /root/anaconda3/envs/gby did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:14<00:00,  7.22s/it]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\"\n",
    "from transformers import AutoModel, AutoModelForCausalLM, LlamaTokenizer\n",
    "from peft import PeftModel, get_peft_model\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained('../DCAI-share/llm/chinese-alpaca-2-7b', trust_remote_code=True, device_map=\"auto\") \n",
    "tokenizer = LlamaTokenizer.from_pretrained('../DCAI-share/llm/chinese-alpaca-2-7b', trust_remote_code=True)\n",
    "# model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "- 食品评价：正面，负面，中性，未提及\n",
      "- 饮品评价：正面，负面，中性，未提及\n",
      "- 价格水平：正面，负面，中性，未提及\n",
      "- 卫生情况：正面，负面，中性，未提及\n",
      "- 服务人员态度：正面，负面，中性，未提及\n",
      "- 停车方便程度：正面，负面，中性，未提及\n"
     ]
    }
   ],
   "source": [
    "from transformers import TextStreamer\n",
    "streamer = TextStreamer(tokenizer,skip_prompt=True,skip_special_tokens=True)\n",
    "\n",
    "text = \"\"\"在三里屯附近搜点评看到这家店，于是过来拔草～店内环境不错，服务态度也很好，点了【安妮披萨】典型的意式薄底披萨，上面铺满培根和香肠，口味说的过去，性价比比较高 【金枪鱼意面】服务员会特意问一下意面要不要煮的软一点，因为意面本身是偏硬的，不过，还挺好吃的【炸薯条】是那种比较粗的薯条，吃起来感觉很好，喜欢！【奶油蘑菇汤】第一次吃里面都是蘑菇丁的奶油蘑菇汤，意外的很好喝～【脆皮奶酪卷】外皮酥脆，奶酪凉凉的，不是很甜腻，很适合餐后吃哦～\\n这家店整体给好评的，无论是餐点还是服务、都不错！\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\\\"方面1：情感类别，方面2：情感类别，...\\\"\\n输出：\"\"\"\n",
    "\n",
    "# few-shot text:\n",
    "# text = \"\"\"Example 1:\n",
    "# 在三里屯附近搜点评看到这家店，于是过来拔草～店内环境不错，服务态度也很好，点了【安妮披萨】典型的意式薄底披萨，上面铺满培根和香肠，口味说的过去，性价比比较高 【金枪鱼意面】服务员会特意问一下意面要不要煮的软一点，因为意面本身是偏硬的，不过，还挺好吃的【炸薯条】是那种比较粗的薯条，吃起来感觉很好，喜欢！【奶油蘑菇汤】第一次吃里面都是蘑菇丁的奶油蘑菇汤，意外的很好喝～【脆皮奶酪卷】外皮酥脆，奶酪凉凉的，不是很甜腻，很适合餐后吃哦～\\n这家店整体给好评的，无论是餐点还是服务、都不错！\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\"方面1：情感类别，方面2：情感类别，...\"\n",
    "# 输出：食品评价：中性，饮品评价：未提及，价格水平：正面，卫生情况：未提及，服务人员态度：正面，停车方便程度：未提及。\n",
    "\n",
    "# Example 2:\n",
    "# 岛上新开的自助烤肉，价格很亲民，据说是老板试营业，找点感觉，但是对于肉食动物们，58块的价格，还是非常划算的。美团上还有团购，54.5，据说前几天刚开的时候，价格还要优惠呢！朋友说门口等位的，排队排好长呢！\\n菜品质量挺好的，没有因为不高的价格而降低质量，排骨、鸡翅都很好吃。水果、熟食畅吃，碳酸饮料，啤酒畅喝，真心觉得不错！\\n朋友小酌，家庭小聚，都是不错的选择。据说，小朋友去还能免单，老板还是很好的。希望以后正式开业后，生意会更好，东西更好吃。\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\"方面1：情感类别，方面2：情感类别，...\"\n",
    "# 输出：食品评价：正面，饮品评价：正面，价格水平：正面，卫生情况：未提及，服务人员态度：未提及，停车方便程度：未提及。\n",
    "\n",
    "# Example 3:\n",
    "# 口味的话只能说算是还不错的火锅 价格有点偏贵个人觉得应该在人均100左右 肉的味道还可以 给的蔬菜也比较干净 只是锅底相比较而言吃起来只能说 还不错 辣锅里面有一点太油腻了 个人不是太喜欢 整体环境很好 服务态度也是不错 自助的水果就比较少了 不过本身也不是冲着它去的 很喜欢的一点就是有那种类似于芝麻油的东西 个人不太能吃辣 吃川式火锅的时候最喜欢这个 蘸了之后又香但是还带着一点点的麻 有吃辣锅的感觉但不至于受不了 这一点还是好评的\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\"方面1：情感类别，方面2：情感类别，...\"\n",
    "# 输出：\"\"\"\n",
    "inputs = tokenizer(text, return_tensors='pt')\n",
    "inputs = inputs.to('cuda:0')\n",
    "output = model.generate(**inputs, max_new_tokens=128,repetition_penalty=1.1, streamer=streamer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "食品评价：正面，饮品评价：正面，价格水平：正面，卫生情况：未提及，服务人员态度：未提及，停车方便程度：未提及。\n"
     ]
    }
   ],
   "source": [
    "from transformers import TextStreamer\n",
    "streamer = TextStreamer(tokenizer,skip_prompt=True,skip_special_tokens=True)\n",
    "\n",
    "# text = \"\"\"在三里屯附近搜点评看到这家店，于是过来拔草～店内环境不错，服务态度也很好，点了【安妮披萨】典型的意式薄底披萨，上面铺满培根和香肠，口味说的过去，性价比比较高 【金枪鱼意面】服务员会特意问一下意面要不要煮的软一点，因为意面本身是偏硬的，不过，还挺好吃的【炸薯条】是那种比较粗的薯条，吃起来感觉很好，喜欢！【奶油蘑菇汤】第一次吃里面都是蘑菇丁的奶油蘑菇汤，意外的很好喝～【脆皮奶酪卷】外皮酥脆，奶酪凉凉的，不是很甜腻，很适合餐后吃哦～\\n这家店整体给好评的，无论是餐点还是服务、都不错！\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\\\"方面1：情感类别，方面2：情感类别，...\\\"\\n输出：\"\"\"\n",
    "\n",
    "# few-shot text:\n",
    "text = \"\"\"Example 1:\n",
    "在三里屯附近搜点评看到这家店，于是过来拔草～店内环境不错，服务态度也很好，点了【安妮披萨】典型的意式薄底披萨，上面铺满培根和香肠，口味说的过去，性价比比较高 【金枪鱼意面】服务员会特意问一下意面要不要煮的软一点，因为意面本身是偏硬的，不过，还挺好吃的【炸薯条】是那种比较粗的薯条，吃起来感觉很好，喜欢！【奶油蘑菇汤】第一次吃里面都是蘑菇丁的奶油蘑菇汤，意外的很好喝～【脆皮奶酪卷】外皮酥脆，奶酪凉凉的，不是很甜腻，很适合餐后吃哦～\\n这家店整体给好评的，无论是餐点还是服务、都不错！\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\"方面1：情感类别，方面2：情感类别，...\"\n",
    "输出：食品评价：中性，饮品评价：未提及，价格水平：正面，卫生情况：未提及，服务人员态度：正面，停车方便程度：未提及。\n",
    "\n",
    "Example 2:\n",
    "岛上新开的自助烤肉，价格很亲民，据说是老板试营业，找点感觉，但是对于肉食动物们，58块的价格，还是非常划算的。美团上还有团购，54.5，据说前几天刚开的时候，价格还要优惠呢！朋友说门口等位的，排队排好长呢！\\n菜品质量挺好的，没有因为不高的价格而降低质量，排骨、鸡翅都很好吃。水果、熟食畅吃，碳酸饮料，啤酒畅喝，真心觉得不错！\\n朋友小酌，家庭小聚，都是不错的选择。据说，小朋友去还能免单，老板还是很好的。希望以后正式开业后，生意会更好，东西更好吃。\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\"方面1：情感类别，方面2：情感类别，...\"\n",
    "输出：食品评价：正面，饮品评价：正面，价格水平：正面，卫生情况：未提及，服务人员态度：未提及，停车方便程度：未提及。\n",
    "\n",
    "Example 3:\n",
    "口味的话只能说算是还不错的火锅 价格有点偏贵个人觉得应该在人均100左右 肉的味道还可以 给的蔬菜也比较干净 只是锅底相比较而言吃起来只能说 还不错 辣锅里面有一点太油腻了 个人不是太喜欢 整体环境很好 服务态度也是不错 自助的水果就比较少了 不过本身也不是冲着它去的 很喜欢的一点就是有那种类似于芝麻油的东西 个人不太能吃辣 吃川式火锅的时候最喜欢这个 蘸了之后又香但是还带着一点点的麻 有吃辣锅的感觉但不至于受不了 这一点还是好评的\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\"方面1：情感类别，方面2：情感类别，...\"\n",
    "输出：\"\"\"\n",
    "inputs = tokenizer(text, return_tensors='pt')\n",
    "inputs = inputs.to('cuda:0')\n",
    "output = model.generate(**inputs, max_new_tokens=128,repetition_penalty=1.1, streamer=streamer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', '', '']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def predict(prompts):\n",
    "    if isinstance(prompts, str):\n",
    "        prompts = [prompts]\n",
    "    assert isinstance(prompts, list), 'input should be list of text'\n",
    "\n",
    "    tokenizer.padding_side = 'left'\n",
    "    input_tensors = tokenizer(prompts, max_length=1024, padding=True, truncation=True, return_tensors='pt')\n",
    "    prompt_length = input_tensors.input_ids.shape[1]\n",
    "    input_tensors.to('cuda:0')\n",
    "    \n",
    "    # 下面是 InternLM 专属 generate 参数\n",
    "    # outputs = model.generate(**input_tensors, max_new_tokens=200,   # 按照指定格式，输出差不多就这么长，多了就不用输出了\n",
    "    #                         repetition_penalty=1.1,\n",
    "    #                         )\n",
    "    outputs = model.generate(\n",
    "        input_ids = input_tensors[\"input_ids\"].to(model.device),\n",
    "        attention_mask = input_tensors['attention_mask'].to(model.device),\n",
    "            #eos_token_id=(2, 103028), \n",
    "        eos_token_id=tokenizer.eos_token_id,\n",
    "        pad_token_id=tokenizer.pad_token_id,\n",
    "        max_new_tokens=200,\n",
    "        repetition_penalty=1.1,\n",
    "        )\n",
    "    # 过滤掉 prompt 部分\n",
    "    real_outputs = []\n",
    "    for i,output in enumerate(outputs):\n",
    "        # output = output[len(inputs.input_ids[i]):]\n",
    "        output = output[prompt_length:]\n",
    "        real_outputs.append(output)\n",
    "    results = tokenizer.batch_decode(real_outputs, skip_special_tokens=True)\n",
    "\n",
    "    return results\n",
    "\n",
    "text1 = \"\"\"自打开业就想去尝尝，终于拔草啦。环境挺舒适的，感觉很明亮舒服～看环境以为是专门吃炸鸡的地方，看菜单才知道是韩式快餐，就是简单的拌饭、冷面之类的，也有一两个炸鸡菜和芝士火锅。有点小失望，还以为会有芝士排骨呢。他家大麦茶是用小碗盛的，挺有意思的。菜量还可以给料也挺足，口味中规中矩，价格不贵。推荐烤牛肉，牛肉火候适中洋葱是点睛之笔，甜甜的很好吃。点的时候被告知韩式辣酱无骨炸鸡要等15分钟才好，结果这菜是第一个上来的…单吃有点咸，可能是我口味淡。牛肉大酱汤里海鲜贝不少，挺舍得放的。哦对了，看旁边那桌点的海鲜乌冬面不错挺有食欲的，下次可以尝尝。写点评时才知道有买单优惠，早知道点评买单了…还能省12。\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\\\"方面1：情感类别，方面2：情感类别，...\\\"\\n输出：\"\"\"\n",
    "text2 = \"\"\"刚去的时候要等位，不过只等了五分钟就有位子了，服务员看到是团购的态度也非常好，上菜也挺快的，菜的味道不错，主要是环境也好，坐着那吃很有感觉。说实话现在吃饭的地方对于这个价位来说算是做的最好的一家餐厅了！H套餐的干锅花菜(非常好吃但是量少= =)，剁椒鱼头(非常好吃，重口味)，跳跳蛙(非常好吃，重口味)，味道一级棒，排骨汤和蒸蛋就一般般了，不过毕竟是配菜也就无所谓~最后补充一句这里位置非常好，吃了就可以去不夜城水星楼玩，好方便。总之这次吃饭感觉非常好，下次还会常来，yeah~\n",
    "---\n",
    "阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\"方面1：情感类别，方面2：情感类别，...\"\n",
    "输出：\"\"\"\n",
    "text3 = \"\"\"整个徐东这边，大概就这么一家吃印度咖喱菜的店子了。里面的装修比较简单，但是却包装的印度风格明显。菜品看上去量一般般，但是真正吃起来却非常扎实。差不多每人一个主食一个咖喱就差不多了，酒水个人认为很一般，选择也不多，也不便宜。咖喱有偏甜口语的玛莎咖喱，也有比较异域一点的酸乳酪椰香咖喱，当然也有普通的，咖喱主要是吃酱汁。用主食的米饭或者饼子沾着吃。芝士烙饼非常不错，拉丝可以很长，口感也很好。服务员态度还行，就是有的时候要叫好久。\n",
    "---\n",
    "阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\\\"方面1：情感类别，方面2：情感类别，...\\\"\\n输出：\"\"\"\n",
    "res = predict([text1, text2, text3])\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "食品评价：中性，饮品评价：未提及，价格水平：中性，卫生情况：未提及，服务人员态度：未提及，停车方便程度：未提及。\n"
     ]
    }
   ],
   "source": [
    "from peft import LoraModel\n",
    "# lora_path = 'weights/absa_baseline_d1_train-1000-llama2'\n",
    "# model = PeftModel.from_pretrained(model, lora_path)\n",
    "\n",
    "text = \"\"\"自打开业就想去尝尝，终于拔草啦。环境挺舒适的，感觉很明亮舒服～看环境以为是专门吃炸鸡的地方，看菜单才知道是韩式快餐，就是简单的拌饭、冷面之类的，也有一两个炸鸡菜和芝士火锅。有点小失望，还以为会有芝士排骨呢。他家大麦茶是用小碗盛的，挺有意思的。菜量还可以给料也挺足，口味中规中矩，价格不贵。推荐烤牛肉，牛肉火候适中洋葱是点睛之笔，甜甜的很好吃。点的时候被告知韩式辣酱无骨炸鸡要等15分钟才好，结果这菜是第一个上来的…单吃有点咸，可能是我口味淡。牛肉大酱汤里海鲜贝不少，挺舍得放的。哦对了，看旁边那桌点的海鲜乌冬面不错挺有食欲的，下次可以尝尝。写点评时才知道有买单优惠，早知道点评买单了…还能省12。\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\\\"方面1：情感类别，方面2：情感类别，...\\\"\\n输出：\"\"\"\n",
    "# text = \"今天天气真不错\"\n",
    "inputs = tokenizer(text, return_tensors='pt')\n",
    "inputs = inputs.to('cuda:0')\n",
    "output = model.generate(**inputs, max_new_tokens=128,repetition_penalty=1.1, streamer=streamer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<s> 今天天气真不错<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk>'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(output[0])\n",
    "# output[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/gby/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda-11.8/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 118\n",
      "CUDA SETUP: Loading binary /root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /root/anaconda3/envs/gby did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [01:04<00:00, 32.17s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LlamaForCausalLM(\n",
       "  (model): LlamaModel(\n",
       "    (embed_tokens): Embedding(55296, 4096, padding_idx=0)\n",
       "    (layers): ModuleList(\n",
       "      (0-31): 32 x LlamaDecoderLayer(\n",
       "        (self_attn): LlamaAttention(\n",
       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (k_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (v_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (rotary_emb): LlamaRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): LlamaMLP(\n",
       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
       "          (act_fn): SiLUActivation()\n",
       "        )\n",
       "        (input_layernorm): LlamaRMSNorm()\n",
       "        (post_attention_layernorm): LlamaRMSNorm()\n",
       "      )\n",
       "    )\n",
       "    (norm): LlamaRMSNorm()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=4096, out_features=55296, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1,2\"\n",
    "from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer\n",
    "from peft import PeftModel, get_peft_model\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained('../DCAI-share/llm/chinese-alpaca-2-7b', trust_remote_code=True, device_map=\"auto\") \n",
    "tokenizer = AutoTokenizer.from_pretrained('../DCAI-share/llm/chinese-alpaca-2-7b', trust_remote_code=True)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda-11.8/lib64/libcudart.so.11.0\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 118\n",
      "CUDA SETUP: Loading binary /root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /root/anaconda3/envs/gby did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "Loading checkpoint shards: 100%|██████████| 8/8 [01:07<00:00,  8.47s/it]\n"
     ]
    }
   ],
   "source": [
    "# Calculating ppl\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "from transformers import AutoModel, AutoModelForCausalLM, LlamaTokenizer, AutoTokenizer\n",
    "from peft import PeftModel, get_peft_model\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained('internlm/internlm-7b', trust_remote_code=True, device_map=\"auto\") \n",
    "tokenizer = AutoTokenizer.from_pretrained('internlm/internlm-7b', trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    1, 29871, 32135, 33657, 30810, 31227, 33084, 31169, 32559, 30910,\n",
       "           262, 32022, 34183, 32547, 31356, 52489, 30690, 30910, 33284, 30584,\n",
       "         30584, 30584, 44298, 30267],\n",
       "        [    1, 29871, 32135, 33657, 30810, 31227, 33084, 31169, 32559, 30910,\n",
       "           262, 32022, 34183, 32547, 31356, 52489, 30690, 30910, 33284, 30584,\n",
       "         30584, 30584, 44298, 30267],\n",
       "        [    1, 29871, 32135, 33657, 30810, 31227, 33084, 31169, 32559, 30910,\n",
       "           262, 32022, 34183, 32547, 31356, 52489, 30690, 30910, 33284, 30584,\n",
       "         30584, 30584, 44298, 30267]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts = ['今年天气这阿萨德哦发in啊搜地方那仨代发不错！！！哈哈哈哈。'] * 3\n",
    "\n",
    "\n",
    "encodings = tokenizer(texts, return_tensors=\"pt\")\n",
    "encodings\n",
    "# encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[-0.2758,  5.6315,  2.6591,  ...,  1.7060,  0.4519,  0.9935],\n",
       "          [-1.2824, 16.4560,  5.3351,  ..., -0.4916, -3.6268, -0.8812],\n",
       "          [ 1.8424,  2.5318,  8.7555,  ...,  3.1162,  3.0829,  2.4762],\n",
       "          ...,\n",
       "          [ 2.2414,  8.6555, 18.4061,  ...,  2.3168,  1.6600,  2.1325],\n",
       "          [ 1.8043,  6.2455, 18.3456,  ...,  4.4452,  3.1189,  1.9373],\n",
       "          [ 1.7926,  6.4306, 18.2170,  ...,  4.0842,  2.4521,  1.0766]],\n",
       " \n",
       "         [[-0.2758,  5.6315,  2.6591,  ...,  1.7060,  0.4519,  0.9935],\n",
       "          [-1.2824, 16.4560,  5.3351,  ..., -0.4916, -3.6268, -0.8812],\n",
       "          [ 1.8424,  2.5318,  8.7555,  ...,  3.1162,  3.0829,  2.4762],\n",
       "          ...,\n",
       "          [ 2.2414,  8.6555, 18.4061,  ...,  2.3168,  1.6600,  2.1325],\n",
       "          [ 1.8043,  6.2455, 18.3456,  ...,  4.4452,  3.1189,  1.9373],\n",
       "          [ 1.7926,  6.4306, 18.2170,  ...,  4.0842,  2.4521,  1.0766]],\n",
       " \n",
       "         [[-0.2758,  5.6315,  2.6591,  ...,  1.7060,  0.4519,  0.9935],\n",
       "          [-1.2824, 16.4560,  5.3351,  ..., -0.4916, -3.6268, -0.8812],\n",
       "          [ 1.8424,  2.5318,  8.7555,  ...,  3.1162,  3.0829,  2.4762],\n",
       "          ...,\n",
       "          [ 2.2414,  8.6555, 18.4061,  ...,  2.3168,  1.6600,  2.1325],\n",
       "          [ 1.8043,  6.2455, 18.3456,  ...,  4.4452,  3.1189,  1.9373],\n",
       "          [ 1.7926,  6.4306, 18.2170,  ...,  4.0842,  2.4521,  1.0766]]],\n",
       "        grad_fn=<ToCopyBackward0>),\n",
       " torch.Size([3, 24, 55296]))"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs = model(**encodings)\n",
    "outputs.logits, outputs.logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(7.8337, grad_fn=<ToCopyBackward0>), torch.Size([]))"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = model(**encodings, labels=encodings.input_ids)\n",
    "res.loss, res.loss.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.4636, grad_fn=<ToCopyBackward0>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts = [\"\"\"上海财经大学,位于上海市,是中华人民共和国教育部直属的重点大学,位列国家“双一流”、“211工程”、“985工程优势学科创新平台\"\"\"]\n",
    "\n",
    "encodings = tokenizer(texts[0], return_tensors=\"pt\")\n",
    "model(**encodings, labels=encodings.input_ids).loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/gby/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda-11.8/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 118\n",
      "CUDA SETUP: Loading binary /root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/gby/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /root/anaconda3/envs/gby did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [01:07<00:00, 33.52s/it]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "from transformers import AutoModel, AutoModelForCausalLM, LlamaTokenizer\n",
    "from peft import PeftModel, get_peft_model\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained('../DCAI-share/llm/chinese-llama-2-7b', \n",
    "                                             trust_remote_code=True, \n",
    "                                             device_map=\"auto\",\n",
    "                                             output_attentions=True)  # <--- get attentions\n",
    "tokenizer = LlamaTokenizer.from_pretrained('../DCAI-share/llm/chinese-llama-2-7b', trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = \"\"\"冲着芝士汉堡来的，本来要等位的，结果和别人拼的大桌，环境很好，可惜服务员态度很有问题，色拉送上来了都没见有人送餐具，问了两三个服务生才要到两副刀叉，表示菜品味道还不错，服务生就大打折扣了！芝士汉堡料很足，味道很棒，色拉也是份量十足，不过凯撒色拉的面包丁看着像是搅成一坨的硬面包，焗蜗牛很香，不过有一个吃到了不明物体，不知道是香料还是啥。总体口味还不错，服务生还得多加培训才是！生意也不算最忙，但却如此怠慢客人，是在不应该！\\n---\\n阅读上面这段评论，观察以下这些方面：食品评价，饮品评价，价格水平，卫生情况，服务人员态度，停车方便程度。请根据评论对这些方面进行情感分析，具体有四类情感：正面、负面、中性、未提及。请用以下格式给出所有方面的情感：\\\"方面1：情感类别，方面2：情感类别，...\\\"\\n输出：\"\"\"\n",
    "text2 = \"\"\"\"\"\"\n",
    "inputs = tokenizer(text1, return_tensors='pt')\n",
    "tokens = tokenizer.convert_ids_to_tokens(inputs.input_ids[0])\n",
    "outputs = model(**inputs)\n",
    "A = outputs.attentions\n",
    "\n",
    "matrix = A[layer_idx][0].sum(axis=0)\n",
    "matrix = np.array(matrix.tolist())\n",
    "matrix = matrix + matrix.T - np.diag(np.diag(matrix))\n",
    "\n",
    "row = matrix.sum(axis=1).squeeze()\n",
    "for i in range(len(row)):\n",
    "    print(str(tokens[i])+'\\t'+str(row[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9gAAAJICAYAAACaO0yGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABy3klEQVR4nO3dZ3gUZf/28TMJSUgILUDoIC2hBAhFaujSQ1WKQgCp3hQVRIoignKDIkgXURBvEClSlIReFKQriFQRpZfQCQnpyTwveLJ/lgRImbAJ+X6Og0N35pqZ32yyV+bcuWbGzjAMQwAAAAAAIFXsbV0AAAAAAADPAwI2AAAAAAAmIGADAAAAAGACAjYAAAAAACYgYAMAAAAAYAICNgAAAAAAJiBgAwAAAABgAgI2AAAAAAAmIGADAAAAAGACAjYAAAAAACYgYAMAAAAAYAICNgAAAAAAJiBgAwAAAABgAgI2AAAAAAAmIGADAAAAAGACAjYAAAAAACbIYusCgOdJbGysoqOjbV0GAAAAEuHo6CgHBwdbl4HnGAEbMIFhGAoKCtLdu3dtXQoAAACeIFeuXCpQoIDs7OxsXQqeQwRswATx4drDw0Ourq502AAAAOmMYRgKCwvT9evXJUkFCxa0cUV4HhGwgVSKjY21hOs8efLYuhwAAAA8houLiyTp+vXr8vDwYLg4TMdNzoBUir/m2tXV1caVAAAA4Gnij9m4bw7SAgEbMAnDwgEAANI/jtmQlgjYAAAAAACYgIANAAAea/Xq1fLy8tLRo0fTfFv+/v7y9/dP8+0AAJBWCNgAAGQA8UHXy8tLv//+e4L5hmGoQYMG8vLy0oABA5K9/iVLlmj16tVmlAoAQKZFwAYAIANxdnZWYGBggukHDhxQUFCQnJycUrTepUuXas2aNaktDwCATI2ADQBABtKgQQNt3LhRMTExVtMDAwNVoUIF5cuXz0aVAQAAAjYAABlI69atdffuXe3evdsyLSoqSps2bVKbNm0StI+Li9O3336r1q1bq2LFiqpTp47Gjh2r4OBgS5vGjRvr9OnTOnDggGUY+qPXQkdFRWnSpEmqVauWfHx8NGjQIN2+fTvB9pYsWaLWrVvL29tbvr6+Gj9+vO7du5eg3fLly/XSSy+pUqVKeuWVVxId9g4AQEZDwAYAIAMpXLiwfHx8tG7dOsu0nTt3KiQkRK1atUrQfuzYsfrss89UtWpVvf/+++rYsaMCAgLUp08fyzNg33vvPRUoUEAlS5bU5MmTNXnyZL3xxhtW65kwYYL++usvDR48WK+++qp+/vlnffTRR1ZtZs2apY8++kgeHh4aNWqUmjdvruXLl6t3795Wz5v94YcfNHbsWOXNm1fvvvuuqlatqv/85z+6evWqmW8VAADPXBZbFwAAAJKnTZs2mjp1qiIiIpQ1a1YFBAToxRdfVP78+a3a/f777/rhhx80ZcoUq7PbNWvWVN++fbVx40a1adNGL730kqZPn67cuXOrXbt2iW4zV65c+uabbyzPj42Li9PixYsVEhKi7Nmz6/bt25o3b558fX319ddfy97+wXf4JUuW1EcffaS1a9fq5ZdfVnR0tKZNm6Zy5cpp0aJFlmvGS5curQ8++EAFCxZMi7cMAIBngjPYAABkMC1btlRkZKR+/vlnhYaG6pdffkl0ePjGjRuVPXt21a1bV7dv37b8q1ChglxdXbV///4kb7Nz586WcC1J1atXV2xsrC5fvixJ2rNnj6Kjo9WjRw9LuJakTp06yc3NTTt27JAkHTt2TLdu3VLXrl2tbsjWoUMHZc+ePdnvBQAA6QlnsAEAyGDc3d1Vu3ZtBQYGKiIiQrGxsWrevHmCdufPn1dISIhq166d6Hpu3bqV5G0WKlTI6nWOHDkkyXJ99ZUrVyQ9OGP9MCcnJxUtWtQSxOPbFS9e3Kqdo6OjihYtmuR6AABIjwjYAABkQH5+fvrggw908+ZN1a9f3xJ4HxYXF6c8efJoypQpia7D3d09ydt7+Kz0wwzDSPI6AAB43jFEHACADKhp06ayt7fX4cOH5efnl2ibYsWK6e7du6patarq1KmT4F/ZsmUtbR8e/p0S8We4z5w5YzU9KipKly5dUuHCha3anT9/3qpddHS0Ll26lKoaAACwNQI2AAAZULZs2TRu3DgNGTJEjRs3TrRNy5YtFRsbqy+++CLBvJiYGKvHZ7m4uCT6OK2kqlOnjhwdHbV48WKrs9orV65USEiIGjRoIEny9vaWu7u7li1bpqioKEu7NWvWpGr7AACkB+lyiPi5c+c0Y8YMHTx4UMHBwSpYsKD8/PzUp08fubi4WNodOnRIn332mU6cOCE3Nze1bNlSQ4cOVbZs2WxYPQAAz0aHDh2eOL9GjRrq0qWL5s2bp5MnT6pu3bpydHTUuXPntHHjRr3//vtq0aKFJKlChQpaunSpvvjiCxUvXtxynXdSubu7a8CAAZo9e7b69u2rxo0b6+zZs/r+++9VsWJFtW3bVtKDa63ffvttjR07Vj179lSrVq106dIlrV69mmuwAQAZXroL2FevXlWnTp2UPXt2de/eXTlz5tThw4c1a9YsHT9+XHPnzpUknTx5Ur169VKpUqU0atQoBQUF6ZtvvtG5c+c0f/58G+8FAADpw0cffSRvb28tW7ZM06ZNk4ODgwoXLqy2bduqatWqlnaDBg3SlStXNH/+fN2/f181atRIVsCWpCFDhsjd3V3fffedJk2apJw5c6pz584aNmyYHB0dLe26dOmi2NhYLViwQJMnT5anp6fmzp2rGTNmmLbfAADYgp2Rzu5O8uWXX2ratGkKDAxUmTJlLNNHjhypH3/8UQcOHFDOnDnVr18/nTx5Uhs3bpSbm5sk6YcfftCYMWO0YMEC+fr62moXkMlERETo7NmzKlGihLJmzWrrcgAAAPAEHLshLaW7a7BDQ0MlSXny5LGani9fPtnb28vR0VGhoaHas2eP2rZtawnXktSuXTu5urpqw4YNz7RmAAAAAADSXcCuUaOGJOn999/XyZMndfXqVa1fv15Lly6Vv7+/XF1dderUKcXExMjb29tqWScnJ5UrV04nT560RekAAAAAgEws3V2DXb9+fb311luaN2+etm/fbpn+xhtvaOjQoZKkGzduSJI8PDwSLJ8vXz4dPHgwVTVERkYqNjY2VetA5hEZGam4uDjFxsbyewMAAJDOxcbGKi4uTuHh4YqLi7N1OcggXF1dk9Qu3QVsSSpcuLCqV6+u5s2bK1euXPrll180b9485cuXT927d1dERISkB2esH+Xs7GyZn1LHjh1L1fLIfLJkyaLIyEhblwEAAICniIyMVExMjM6cOWPrUpCBVKtWLUnt0l3AXrduncaOHatNmzapQIECkqRmzZrJMAxNmTJFrVu3ttyM4OHnZ8aLjIxM9c0KvL29OROJJIuMjNSVK1fk7OzMjTIAAAAygCxZsqhYsWJydna2dSl4zqS7gP3999+rXLlylnAdr3Hjxlq9erVOnjypfPnySZKuX7+eYPkbN24kOnQ8OfigITns7e1lb28vBwcHOTg42LocAAAAPIGDg4Ps7e3l4uLCyRGYLt3d5OzmzZuJXgsRHR0tSYqJiZGnp6eyZMmSYCh3VFSUTp48qbJlyz6TWgEAAAAAiJfuAnaJEiV04sQJnT171mr6unXrZG9vLy8vL2XPnl21a9fW2rVrLY/1kqSffvpJYWFhatGixbMuGwAAAACQyaW7IeJ9+vTRzp071a1bN3Xr1s1yk7OdO3eqU6dOyp8/vyRp6NCh6tq1q/z9/dW5c2cFBQVp4cKF8vX1Vf369W28FwAAAACAzMbOMAzD1kU86siRI5o1a5ZOnjypu3fvqnDhwurQoYP69u2rLFn+7zuB33//XVOmTNGJEyeULVs2tWzZUsOGDZObm5sNq0dmExERobNnz6pEiRJcxwMAAJDOceyGtJQuAzaQkdBJA3jYqFGjdOzYMQUGBqb5to4fP67x48fr1KlTioiI0G+//ab27durYcOGGjt2bKrWfenSJa1Zs0adO3e2jB5D8mzdulXXrl1Tt27dLNM2bNigtWvX6vjx47p3756KFy8uf39/vfzyy7Kzs7O0mzVrlurWrauqVavaonSkULt27VSuXDl98sknSV7m0qVLatKkiWbMmGG5zLFx48amfI4zm8Q+c4nh2A1pKd1dgw3AdmbNmiUvLy/Lv4oVK6ply5b6+uuvE7354LOwf/9+eXl56ejRo5ZpXl5eWrBgwTOtIzIyUg0aNNAvv/ximda4cWN99NFHz2T7s2bNUpUqVUxf77fffqsdO3ZYTYuLi1Pz5s21du1a07eXGQwcOFBTpkx5JtuaMGGCYmNjNW/ePC1fvlzZsmXT7Nmz1bt371Sv+/Lly5o9e3aiT+xA0mzdulVLly61mvbtt9/KxcVFo0aN0ty5c1W/fn198MEHmjNnjlW72bNn648//niW5SIdMetznNkk9pkDnrV0dw028DyJizNkb2/39IbpaNtZs2bV//73P0kPvuHdv3+/pk6dKsMw1L9/f7PLTJHly5erUKFCz3SbS5cuVY4cOdSwYUPLtNmzZytHjhzPtA6zLVq0SA0bNlSDBg0s0+zt7dW/f3/NmjVLrVq1sro0xxaMuDjZ2dvm++CUbLtYsWJpVE1CZ86c0WuvvaZatWpZppUvX/6JyxiGoejoaDk5OaV1eaaLi4uTvQ1+F8zc7ty5c+Xu7m55Xbt2bd29e1cLFy7UwIEDbbJ/SWGrz6EtP/+29LTPcXqRkfsTIK0QsIE0ZG9vpzlLd+vy9eBnut3CHjk16NW6KVrW3t5ePj4+lte1atXS33//rc2bN6ebgP1wfc+CYRhatGiRevToYTU9oxwApUSrVq00YcIE/fLLL3rppZdsWoudvb3OBn6t8FtXn+l2XfIUVAm/fsle7uEh4qtXr9bo0aO1Zs0aff755/r999/l4eGhgQMHqn379pZlDh48qM8//1x//fWX4uLiVKRIEfXu3VsdOnRIdBv79++3/D5+8cUX+uKLL1SjRg0tXrw4wdDS+HreffddTZ06VWfOnNGUKVPUpEkTff7559qwYYNu3rypXLlyydvbW5999plOnDhhWf8rr7xi2e6pU6eS/X6Yyd7eXvN2LNKV4GvPbJuFcubXgAY9nt7wEaNGjdKaNWskPRh1I0kdOnRIdOhwuXLltGLFCoWFhcnNzc3SfvLkyZo8ebKkB1+G1axZM6W7kWp29vY6PHeeQq88u8+hW6GC8vnPgBQtG/97/95772nSpEk6f/68KlWqpE8++URubm768MMP9euvv8rd3V3Dhg1Tq1atrJZftmyZFi5cqMuXL8vDw0OvvPKK3njjDasvQA4dOqQJEybo9OnTKl68uN59991Ea/njjz80bdo0HTlyRA4ODmrYsKHee+895cmT57H1P+5z/MEHH2jSpEk6d+6cSpcurXHjxsnb29uyXEhIiMaPH69t27Ypa9as6tSpk3LlyqVPP/30iZ/f+L5q+fLl+vzzz/Xnn38qT548GjhwoFUf8Lj+pEWLFtq8ebPmzJmjM2fOKGfOnGrdurWGDRsmZ2dnSf/Xb82fP18rV67Uzp07lTNnTr3zzjtq06aNFi1apG+++UZhYWFq2rSpPvzwQ0twT0p9yfnMAWmJgA2kscvXg3Xu8h1bl5Eq2bJlU0xMjNW0KVOmaMeOHbp06ZLc3Nz04osvatSoUfLw8LC0SUpo+OWXXzRnzhydOnVKrq6uat68uUaOHClXV9fH1uPl5aURI0aoT58+kiR/f3+5urqqQ4cOmjZtmq5fv66KFStqwoQJVmcTo6KiNHv2bAUEBOjGjRsqWrSoBg4cqDZt2jxx/w8cOKDLly+refPmVtNTegC0cuVKLVy4UBcvXpSLi4tKliyp0aNHq1KlSk+s42FhYWGaMmWKdu/eraCgIOXJk0e+vr569913lT17dku7bdu2ac6cOTp79qwcHBxUrFgxvfXWW2rQoIEaN26sy5cva8mSJVqyZIkkadKkSerYsaNcXFzUoEEDrVmzxuYBW5LCb11V+LULti4jxYYPH67OnTvr9ddf14oVKzRq1ChVrFhRpUqVUmhoqAYMGKBq1arp888/l5OTk/755x/du3fvseurUKGCli9frtdff12tWrVSp06dnniDz+vXr2vChAn6z3/+o4IFC6pQoUKaN2+eli1bpuHDh6tMmTK6c+eOdu/eraioKFWoUEFjx47VRx99pEmTJqlkyZJp8bakyJXgazp/65Kty3iqgQMH6vbt25YAIsnqzPXDDh48qPz581t+hsuXL1eXLl3k7+8vPz8/SVLp0qWfTeFPEHrlqu6dP2/rMpLsxo0b+uSTT/Sf//xHWbJk0YQJEzR8+HC5uLioevXq6ty5s1asWKF3331XlStXVuHChSVJixcv1oQJE+Tv76+GDRvqjz/+0OzZsxUSEqKRI0da1t2nTx95eXlp+vTpunfvnsaPH6+wsDCVK1fOUsMff/whf39/NWjQQNOmTVN4eLimT5+ugQMHavny5cnenwkTJqh///7Knj27pk6dqsGDB2vLli1ydHSUJI0ePVr79u3Tu+++q8KFC2vFihU6fvx4krcxbNgwdenSRf369dP69ev1/vvvy8PDw+oJPYn1J9u2bdObb76p1q1b65133tGZM2c0bdo0Xb16VTNnzrTaxrhx49ShQwfL+z9ixAj99ddfOn36tMaPH6+LFy/qk08+UdGiRfXGG28kub7kfOaAtETABpBAfJiOHyK+efNmDRhgfRbh1q1bGjBggDw8PHT79m0tXLhQ/v7+WrdunbJkyZKk0LBx40YNHTpUHTt21JAhQ3Tjxg1NnTpV9+7d07Rp05JV88mTJ3X79m0NHz5csbGx+uSTT/Tuu+9aHcC89dZbOnTokAYNGqRSpUppx44devfdd5UjRw6rIdKP2rNnjwoWLKiCBQs+tY6nHQD99ttvev/999W7d281aNBAEREROnLkiEJCQpK1vxEREYqNjdXQoUPl7u6uq1ev6ssvv9TAgQO1ePFiSdKFCxf01ltvWQ544uLi9Ndffyk4+MGIitmzZ6t///6qWrWq5Vq/h7+QqFKlimbOnGmzYbnPk/hHT0oP3tcdO3Zo06ZNGjhwoM6ePauQkBANGzbMctaldu3aT1yfm5ubfHx85ODgoAIFCjx1VEdwcLC+/vprVa5c2TLt6NGj8vX1tboZ0MNfIsUHujJlyqhixYrJ2l88+Cy5u7vrypUrT/z5/P7771q/fr0luEn/N0qnYMGCz3zEzvMkODhY3333ncqUKSPpQTD8+OOP1a9fPw0aNEiSVLFiRW3ZskVbt25Vz549FRsbqzlz5qh169YaM2aMJMnX11fR0dH65ptv1L9/f+XOnVv/+9//ZGdnp6+//trypWaBAgXUq1cvqxqmTp0qb29vzZ4923ITO09PT/n5+WnHjh1P/NvztP1xcXFRjx499Oeff6p69er6559/tGXLFn366aeWETL16tVTy5Ytk7yNdu3aWf7e16tXTxcvXtScOXOsAnZi/cnbb78tHx8fTZ06VZJUv359ubi4aOzYsTp16pSlb5OkFi1aaPDgwZKkSpUqacuWLVq3bp3VFwUHDhzQxo0bEwTsJ9WX1M8ckNYI2ACshIWFqUKFClbTWrVqlWB4+KRJkyz/HxsbqypVqqh+/frat2+ffH19nxoaDMPQ5MmT1apVK/33v/+1TM+XL5/69++vgQMHWg4ikiIkJEQ//vij5dvqsLAwjR49WkFBQSpQoID27dun7du3a8GCBfL19ZUk1a1bVzdu3NCsWbOeeJBz7Ngxq4ODJ3naAdCRI0eUK1cuq4Pph6/rTip3d3eNHz/e8jomJkZFihTRa6+9Zrkz6okTJxQdHa0PPvjAcmasXr16lmXKly8vJycn5c2bN9GDkbJlyyo0NFT//vtvsn4WSCj+d06SXF1dVahQIQUFBUl6EMTc3Nw0btw4+fv7q1atWlZnXQzDUGxsrOW1vb19sr/wyJUrl9XBsPTg579gwQLL77+3tzdfpDxjQUFBGjp0qGrWrJngEhSknoeHh1Xf9cILL0iS6tSpY5mWI0cOubu7Wz6PZ86c0Z07dyx3847XqlUrzZs3T0eOHFGDBg30559/qmbNmlYjhmrXrq1cuXJZXoeHh+vQoUMaMWKE1Wf4hRdeUMGCBXX06NFkBexH9yf+S7Br1x5cMhF/M9AmTZpY2tjb26tRo0ZauHBhkrbRtGlTq9fNmjXT5MmTFRsbKwcHB0kJ+5P79+/r5MmTVn/XpAfv2dixY3Xw4EGrv6F16/7fJWzZs2eXu7u7qlevbgnX0oP3aP/+/SmqD7A1/pICsJI1a1atXLlSK1eu1Pfff6/3339fv/76q+Wb/Hg7duxQ165dVa1aNZUvX97y7fa5c+ckWYeG9evX6/bt21bLnz17VpcvX1bLli0VExNj+VejRg3Z29vr2LFjyaq7bNmyVqEk/sAj/qBp9+7dypUrl2rVqmW1vTp16ujkyZNWBz+Pun79epKHmT3tAKh8+fK6e/euRo0apd27dys8PNxq+bi4OKv6nvQkxR9//FHt27dXlSpVVKFCBb322muS/u9n4OXlJQcHBw0fPlzbt29P9lny3LlzS3pwVh6p8/BBuCQ5OjoqKipKkpQzZ04tXLhQ2bJl04gRI1S3bl35+/tbrpdcs2aNKlSoYPn33nvvJXv7efPmTTDtP//5j/r166c1a9aoU6dOqlu3rmbPnv3E3zmY5969e+rXr59y5cqlWbNm8eVGGnj0JpTxAe7Rz6OTk5MiIyMlyTLC59Hro+Nfx8+/ceNGotdQP/y34t69e4qNjdWkSZOsPsMVKlTQlStXdPVq8q5nf9z+xNd+48YNOTo6Jti/5AyTfnSf8ubNq+joaN25c8dq2sNCQkJkGEaCZbNnzy4nJyfLe/bw9Ic5OTklum/xfWRy6wNsjTPYAKzY29tbDQetVq2aZcj166+/Lk9PTx05ckQDBw5UkyZN1K9fP+XJk0d2dnbq3Lmz5Q99fGiYOXOm5dv76tWra8yYMfLy8rL8MYwfpvcosw887ty5o7t37yY4Ox/vxo0bKlCgQKLzoqKirL5ZT00dtWvX1uTJk7Vo0SL16dNHzs7Oat68ud577z3lypVL7733nuUmLdL/XRP9qC1btmjkyJHq0qWLhg4dqly5cunGjRsaNGiQZVslSpTQl19+qXnz5mnw4MGyt7eXr6+vxo4dm6S7sMffXCYiIiJJ+46Uq1SpkubPn2+5LOPTTz/VoEGDtHXrVjVq1EgrV660tI3/4iM5Hn6+cjwnJycNGTJEQ4YM0fnz57Vq1SrNmjVLRYoUsboBG8wXERGhAQMGKCQkRMuXL08QOGA78WegH/1S+NatW5Ie/G2THoy2ip/2sIeXy549u+zs7DRgwIBE72WRks/yk+TLl0/R0dEKCQmx+p16dF+e5NatW1bPvb9586YcHR2tan20P4nfz0e3ExISoqioKMt7Zoak1AfYGgEbwFPF3+Don3/+kaenp7Zu3So3NzdNnz7dctbl8uXLCZZ7UmiIP4gZO3Zsojf3evhmaWbImTOn3N3d9dVXXyU6/0nf8OfMmTPZZ3+fpF27dmrXrp1u376tbdu2adKkScqSJYsmTpyowYMHW10TW6RIkUTXsXHjRpUrV87qOdwHDhxI0K5+/fqqX7++QkNDtXPnTk2aNEmjR4+2PIrtSeKvl394yCPSVtasWdWgQQNduHBB//3vfxUZGancuXOn+cFj8eLFNWzYMC1fvlxnzpyRlPDLISSfo6NjgvcvJiZGb7/9ts6cOaMlS5ZYhYWnLYu0V6JECbm7u2vjxo1Ww5E3bNggR0dHy9+rSpUqaenSpVZhdu/evbp7965lGVdXV/n4+OjMmTPP5D4G8TfT3LZtm+VLsri4OP38889JXseWLVusnpCxefNmVahQ4YnDr7Nly6Zy5cpp48aNVtegb9iwQdKDL+rN8rT6+NwgPSBgA3iq06dPS/q/b9sjIiLk6Oho9S12QEDAY5dPLDSULFlSBQoU0MWLF60CZVqpU6eO5s+fL0dHR5UtWzZZy5YoUcISOszk7u6uTp06aefOnZb1FylS5LGh+mHxP4OHPeln4ObmplatWunIkSMKDAy0TH/SwUj8lybx1y0ibfzyyy9auXKlXnrpJRUqVEg3b97Ud999p6pVq1oeb5MWBg4cqAoVKqh8+fJycXHRzz//rODgYMsztV944QU5ODho1apVypIlixwcHLjZWTKVKlVKq1atUmBgoIoXL67cuXNr3rx5+vnnnzVq1CiFhobq8OHDlvbx90WQHnyxuW3bNlWvXl0uLi4qUaLEE+8UD3M4ODho4MCBmjBhgtzd3dWgQQMdPnxYX3/9tXr27Gn5O9izZ099//336tevn/r166d79+5p1qxZCb6QHDFihHr27Km3335brVu3Vo4cORQUFKQ9e/aoY8eOpj56rUyZMmratKkmTJig8PBwFSpUSCtWrFBERESio1gS89NPPylr1qwqX7681q9fr99+++2xX0w/bPDgwRo0aJCGDx+utm3b6uzZs5o2bZqaN2+e5HuYmFFfYp+5pPxNBcxEwAbSWGEP84ZGPYttxsXFWQ74oqOjdfz4cc2dO1elS5dW9erVJT24Qcn//vc/ffzxx2ratKn++OMP/fTTT1brSUpoGDVqlIYPH66wsDA1bNhQLi4uunLlinbs2KGhQ4eqRIkSKd6PR9WtW1eNGjVS37591bdvX3l5eSk8PFz//POPzp8/b3WjtUdVrVpVGzZsUHR0dJKHij/OzJkzdffuXdWoUUN58uTR33//rV9//TXBnWefpk6dOvroo480Z84cy12p9+7da9Vm2bJlOnz4sOrVq6d8+fLp0qVLWrt2rdUNZkqWLKl9+/Zp9+7dypEjh4oUKWI5gDx27JhKlSqVLh5z4pLn6Xdwz6jbLFasmOzt7TV9+nTdunVLuXLlkq+vr4YNG5am243/vV64cKFiY2NVokQJTZkyxXIDKHd3d40dO1bz58/X2rVrFRMTY/PnYEsPnkudUbb3yiuv6MiRI/r444919+5ddejQwTLSJLFn827bts0SBsaOHauJEyeqX79+ioiIsPlzsKUHz6V+nrcXz9/fX1myZNG3336rpUuXKl++fBo8eLDVHa09PDz09ddfa8KECXrrrbdUrFgxjR07NsETMKpWrarvv/9es2bN0ujRoxUdHa0CBQqoVq1aKl68uOm1T5w4UR999JEmT54sJycndejQQWXKlLE8ivFppk6dqs8//1xz5sxRnjx59PHHHyfpRmxNmjTRjBkzNGfOHA0cOFC5cuVS586d9c4776R2l5JVX2KfOZ6DjWfOAJAq4eHhxokTJ4zw8PAE82Jj42xQUcq3PXPmTMPT09Pyr3z58kbjxo2NcePGGTdv3rRq+9VXXxn169c3KleubLz++uvG2bNnDU9PT2P+/PmGYRjGv//+awwZMsRo0KCB4e3tbfj6+hqjRo0yrl+/brWeXbt2Gd27dzd8fHwMHx8fo3Xr1sYnn3xi3Lt3zzAMw9i3b5/h6elpHDlyxLLMw9sxDMPo3r270b9/f6v1njhxwvD09DT27dtnmRYZGWnMmjXLaNasmVGhQgWjVq1ahr+/v7FmzZonvi83btwwypcvb+zatctqeqNGjYzx48dbXo8cOdJo3bq1VZvg4GDD09PTWLVqlWEYhrF9+3ajZ8+eRq1atQxvb2/jpZdeMmbOnGlER0c/sYaZM2caPj4+ltcxMTHGJ598YtSqVcuoUqWKMWTIEOPw4cOGp6ensWHDBsMwDOPQoUNG//79jbp16xoVKlQwGjZsaEyYMMEICQmxrOfvv/82XnvtNaNKlSpWdRqGYfj5+RnTp09/Yl3PQlxsbKbcNhKKtdHPw1bbTU9s9VngM5h6r732mtG9e/cntlm1apXh6elp3Lp16xlVlTxm1/ekYzcgtewMg9uFAqkRERFheSxS1qxZbV0O0siQIUPk5uZm9Xiy59np06fVrl07bdq0SUWLFrV1OQCAJNi0aZOuXr0qT09PhYeHKzAwUOvXr9ecOXMSvdFavNWrV2v06NHau3dvuhi19Ciz6+PYDWmJIeIAkAQDBw7Uq6++qnfeeSfRRx49b7755hu1a9eOcA0AGYirq6t++uknnTt3TtHR0SpZsqQ+++yzJ4ZrAObiDDaQSnwLmnmsWLFC5cqVe+5v9BQXF6evvvpK7du3f+yjywAAyKg4dkNaImADqUQnDQAAkHFw7Ia0ZG/rAgAAAAAAeB4QsAGTMBgEAAAg/eOYDWmJgA2kUpYsD+4VGBMTY+NKAAAA8DTxx2zxx3CAmQjYQCo5ODjIwcFB9+7ds3UpAAAAeIp79+5Zjt8As/G1DZBKdnZ28vDw0NWrV+Xs7Kxs2bLJzs7O1mUBAADgIYZh6P79+7p3754KFizI8RrSBHcRB0xgGIaCgoIUHBzMdT0AAADplJ2dnXLmzKkCBQoQsJEmCNiAiWJjYxUdHW3rMgAAAJAIR0dHhoYjTRGwAQAAAAAwATc5AwAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAJ5TRlycrUt4pjLb/iL9sTMMw7B1EQAAAADSxtnArxV+66qty0hzLnkKqoRfP1uXgUwui60LAAAAAJB2wm9dVfi1C7YuA8gUGCIOAAAAAIAJCNgAAAAAAJiAgA0AAAAAgAkI2AAAAAAAmICADQAAAACACQjYAAAAAACYgIANAAAAAIAJCNgAAAAAAJggi60LeNSoUaO0Zs2ax87fuXOn8ufPL39/fx04cCDBfF9fXy1YsCAtSwQAAAAAIIF0F7C7dOmi2rVrW00zDEPjxo1T4cKFlT9/fsv0AgUKaNiwYVZtPTw8nkmdAAAAAAA8LN0F7CpVqqhKlSpW037//XeFh4erTZs2VtOzZ8+udu3aPcvyAAAAAABIVIa4BjswMFB2dnby8/NLMC8mJkb379+3QVUAAAAAAPyfdHcG+1HR0dHasGGDqlSpoiJFiljNO3funHx8fBQdHa28efOqU6dOGjRokBwdHW1ULQAAAAAgs0r3AXvXrl26e/duguHhRYsWVc2aNeXp6amwsDBt2rRJc+fO1blz5zR9+vRUbTMyMlKxsbGpWgcAAABgS3Z2dnJxcbF1Gc9ceHi4DMOwdRl4zri6uiapXboP2IGBgXJ0dFTLli2tpk+cONHqdfv27fXBBx9oxYoV6tWrl3x8fFK8zWPHjqV4WQAAACA9cHFxUfny5W1dxjN39uxZhYeH27oMPGeqVauWpHbpOmDfv39f27Ztk6+vr3Lnzv3U9q+//rpWrFihPXv2pCpge3t7cwYbAADgOWNnZ2frEp6pzLa/8UqUKMEZbNhMug7YW7duTfTu4Y9TsGBBSVJwcHCqtuvs7Jyq5QEAAJD+xMUZsrfPnKEzM8mMw+KRfqTrgB0QECBXV1c1btw4Se0vXrwoSXJ3d0/LsgAAAJAB2dvbac7S3bp8PXUnYzKKyl6F1KWFj63LADKVdBuwb9++rb1796p169YJvoUKDQ2Vk5OTnJycLNMMw9DcuXMlSb6+vs+0VgAAAGQMl68H69zlO7Yu45kolC+HrUsAMp10G7DXr1+vmJiYRIeHHz9+XO+8845at26tYsWKKTIyUlu2bNGhQ4fUpUsXVahQwQYVAwAAAAAys3QbsAMCApQnTx7VqVMnwbxChQqpWrVq2rJli27evCl7e3uVLFlS48ePV5cuXWxQLQAAAAAgs0u3AXv58uWPnVe0aFHNmDHjGVYDAAAAAMCT2du6AAAAAAAAngcEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGACCJjLg4W5fwTGW2/QUAILWy2LoAAAAyCjt7e50N/Frht67aupQ055KnoEr49bN1GQAAZCgEbAAAkiH81lWFX7tg6zIAAEA6xBBxAAAAAABMQMAGAAAAAMAEBGzYTGa8eU5m3GcAAAAgs+AabNhMZrpZkMQNgwAAAIDnHQEbNsXNggAAAAA8LxgiDgAAAACACQjYAAAAAACYgIANAAAAAIAJCNgAAAAAAJiAgA0AAAAAgAkI2AAAAAAAmICADQAAAACACQjYAAAAAACYgIANAAAAAIAJCNgAAAAAAJiAgA0AAAAAgAkI2AAAAAAAmICADQAAAACACQjYAAAAAACYgIANAAAAAIAJCNgAAAAAAJiAgA0ASLG4OMPWJQAAAKQbWWxdAAAg47K3t9Ocpbt1+XqwrUtJc5W9CqlLCx9blwEAeIws2XIoLi5O9vaZ5xxiZtvfjICADQBIlcvXg3Xu8h1bl5HmCuXLYesSAABPkMXZVfb29pq3Y5GuBF+zdTlprlDO/BrQoIety8AjCNgAAAAAnhtXgq/p/K1Lti4DmRTjCQAAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMEGWlCx08eJF7du3T4cOHVJQUJDu3LmjrFmzyt3dXZ6enqpRo4aqV68uJycns+sFAAAAACBdSnLANgxD69at07Jly3Tw4EHLtEdt375d8+bNU44cOdSxY0e99tprKlq0qHkVAwAAAACQDiUpYO/cuVOfffaZTp8+rdy5c+uVV16Rj4+PvL29lTdvXuXMmVMREREKDg7W2bNn9eeff2r37t369ttv9d133+m1117TwIEDlTNnzrTeHwAAAAAAbCJJAbt///6qVq2a5s6dq3r16ilLloSLubm5yc3NTYULF5avr68GDRqky5cva8WKFVqyZImyZ8+uwYMHP3Vbo0aN0po1ax47f+fOncqfP78k6dChQ/rss8904sQJubm5qWXLlho6dKiyZcuWlN0CAAAAAMA0SQrYCxcuVO3atZO98sKFC2vo0KHq06ePLl26lKRlunTpkmBbhmFo3LhxKly4sCVcnzx5Ur169VKpUqU0atQoBQUF6ZtvvtG5c+c0f/78ZNcKAAAAAEBqJClgpyRcPyxHjhwqX758ktpWqVJFVapUsZr2+++/Kzw8XG3atLFM+/zzz5UjRw4tXrxYbm5ukqQiRYpozJgx2rVrl3x9fVNVsy3ExRmyt7ezdRkAAAAAgBRI0V3En7XAwEDZ2dnJz89PkhQaGqo9e/aoZ8+elnAtSe3atdPEiRO1YcOGDBmw7e3tNGfpbl2+HmzrUtJcZa9C6tLCx9ZlAAAAAIBpTAnYQUFBWrp0qc6cOSM7OzuVKlVKXbt2tQznTo3o6Ght2LBBVapUUZEiRSRJp06dUkxMjLy9va3aOjk5qVy5cjp58mSqt2srl68H69zlO7YuI80VypfD1iUAAAAAgKlSHbB37typwYMHy8nJSSVKlFBUVJS2bdumhQsXau7cuakeXr5r1y7dvXvXanj4jRs3JEkeHh4J2ufLl8/yGLGUioyMVGxsbKrWkVx2dnZycXF5ptuEbYSHhyf6iDsgo6Hfyhzos/C8oM/C84p++tlwdXVNUrtUB+xJkyapadOmmjhxopydnSVJly9fVq9evTRp0iStXbs2VesPDAyUo6OjWrZsaZkWEREh6cEZ60c5Oztb5qfUsWPHUrV8Sri4uCT5OnVkbGfPnlV4eLitywBSjX4rc6DPwvOCPgvPK/rpZ6NatWpJapfkgP3VV1+pb9++sre3t5p+4cIFffjhh5ZwLT24e3jjxo31/fffJ3X1ibp//762bdsmX19f5c6d2zI9a9askqSoqKgEy0RGRlrmp5S3t7dNzmAjcyhRogTfMuK5QL+VOdBn4XlBn4XnFf10+pLkgL1gwQJt2LBBEyZMUIUKFSzTixcvrpUrV6pq1aqWM8pXr17V9u3b9cILL6SquK1btya4e7j0YBi4JF2/fj3BMjdu3Eh06HhyPPxlAWA2hqcByEjoswAgfaOfTl/sn97kgXXr1qlYsWLq0qWLPv30U8sw7JEjR2rTpk2qW7euOnfurPbt26tp06a6du2aRo4cmariAgIC5OrqqsaNG1tN9/T0VJYsWRIM5Y6KitLJkydVtmzZVG0XAAAAAIDkSnLAzps3r2bMmKGZM2dqw4YN8vPz0549e9SgQQNt3LhRXbp0kYeHh4oUKaI+ffpo06ZNqXpU1u3bt7V37141bdo0wbcy2bNnV+3atbV27VqFhoZapv/0008KCwtTixYtUrxdAAAAAABSItk3OWvcuLFq1KihKVOmqG/fvmrbtq1Gjx6t4cOHm1rY+vXrFRMTk2B4eLyhQ4eqa9eu8vf3V+fOnRUUFKSFCxfK19dX9evXN7UWAAAAAACeJslnsB/m5uamcePGafHixTp69KhatmypwMBAUwsLCAhQnjx5VKdOnUTnV6hQQQsXLpSzs7MmTZqkFStW6JVXXtGMGTNMrQMAAAAAgKRI1WO6qlWrph9//FFffPGFRo0apYCAAI0bN04FCxZMdWHLly9/apvq1atr2bJlqd4WAAAAAACplawz2Fu3blW/fv3k5+enfv36aevWrXJ0dNRbb72lNWvW6O7du2rdurUWLVqUVvUCAAAAAJAuJTlg//DDDxoyZIguXryoMmXK6NKlSxoyZIhWrVolSSpTpoyWLVumt99+W9OnT1eXLl10+vTpNCscAAAAAID0JMkBe+HChapSpYrWr1+vadOmad26dfLx8dGCBQssbezs7NSjRw8FBgYqZ86c6tChQ5oUDQAAAABAepPkgB0UFCQfHx/Z2z9YxN7eXlWqVFFQUFCCtoUKFdJXX32lTz75xLxKAQAAAABIx5IcsMuUKaMtW7bo2rVrkqRr165p69atKlOmzGOX8fPzS32FAAAAAABkAEm+i/iIESPUt29fNW7cWLlz59bdu3eVJUsWTZo0KS3rAwAAAAAgQ0hywK5WrZrWr1+vtWvXKigoSAULFpSfn58KFSqUlvUBz40s2XIoLi7OcplFZpDZ9hcAAACZW7Keg12wYEENGDAgrWoBnmtZnF1lb2+veTsW6UrwNVuXk+YK5cyvAQ162LoMAAAA4JlJVsAGkHpXgq/p/K1Lti4DAAAAgMmSNHbzq6++Unh4eIo3cvjwYf3yyy8pXh4AAAAAgPQuSQF77ty5atKkiWbPnq0LFy4kacVRUVHauHGj+vTpo1dffVVnzpxJVaEAAAAAAKRnSRoivnHjRk2fPl1ffPGF5syZo3Llyqly5cqqUKGC8ubNq+zZsysqKkp3797V2bNndeTIER08eFChoaEqXLiwpk6dqlatWqX1vgAAAAAAYDNJCtj58+fXpEmT9MYbb2jZsmX66aefdOLECdnZ2SVoaxiG7O3t9eKLL6pr165q2rSpsmThUm8AAAAAwPMtWcm3ePHiGjlypEaMGKFTp07p0KFDunbtmu7cuaOsWbPK3d1dnp6eql69unLkyJFWNQMAAAAAkO6k6NSynZ2dypYtq7Jly5pdDwAAAAAAGVKSbnIGAAAAAACejIANAAAAAIAJCNgAAAAAAJiAgA0AAAAAgAkI2AAAAAAAmICADQAAAACACQjYAAAAAACYIEXPwY4XFRWlvXv36syZMwoLC9OgQYMkSZGRkQoNDVXu3Lllb0+GBwAAAAA8/1Kcfrdt26ZGjRrpjTfe0KeffqrZs2db5p06dUq+vr5at26dKUUCAAAAAJDepShgHzx4UG+99ZacnJz0/vvvy8/Pz2p+pUqVVKxYMW3evNmUIgEAAAAASO9SNET8iy++UPbs2bVq1Sq5u7vr7t27Cdp4e3vryJEjqa0PAAAAAIAMIUVnsI8cOaImTZrI3d39sW0KFiyomzdvprgwAAAAAAAykhQF7KioKLm5uT2xzb1792RnZ5eiogAAAAAAyGhSFLCLFi2qo0ePPrHN4cOHVbJkyRQVBQAAAABARpOigN2sWTMdOnRIq1atSnT+ggULdPr0abVq1SpVxQEAAAAAkFGk6CZnffr00ebNmzVmzBgFBgYqKipKkjR58mQdPnxYf/zxh8qVK6fu3bubWiwAAAAAAOlVis5gZ8uWTUuWLFGrVq104MABHTx4UIZh6JtvvtEff/yhli1bauHChXJycjK7XgAAAAAA0qUUncGWpJw5c2rq1KkaM2aMjh49quDgYLm5ualixYrKmzevmTUCAAAAAJDupThgx8udO7fq169vRi0AAAAAAGRYKRoiDgAAAAAArKXoDHaPHj2e2sbe3l5ubm4qUaKEXnrpJVWuXDklmwIAAAAAIENIUcA+cOCAJMnOzk6GYSSY/+j0+fPnq2PHjvrvf/+bwjIBAAAAAEjfUjRE/MiRI2rUqJFKlSqlqVOn6ueff9aRI0f0888/a+rUqSpdurQaN26sHTt26JtvvlH58uW1evVqff/992bXDwAAAABAupCigD1z5kz9/fffWrFihVq3bq2CBQvKyclJBQsWVOvWrbV8+XKdOnVKixcvVp06dbRw4UK5u7tr9erVZtcPAAAAAEC6kKKAHRgYqGbNmsnV1TXR+a6urmrWrJnWrVsnScqRI4fq1aunf//9N+WVAgAAAACQjqUoYN++fVvR0dFPbBMTE6Nbt25ZXufLl09xcXEp2RwAAAAAAOleigJ2sWLFtHHjRt25cyfR+Xfu3NGGDRtUrFgxy7Tr168rZ86cKasSAAAAAIB0LkUBu3v37rp586Y6duyoRYsW6dixY7p69aqOHTumRYsWqWPHjrp165a6d+8uSYqLi9O+fftUsWJFU4sHAAAAACC9SNFjurp27apr167pq6++0qRJk6zmGYYhe3t79e/fX127dpUk3b17V71791aVKlVSXzEAAAAAAOlQigK2JL311ltq166dAgMDderUKYWGhsrNzU1eXl5q3bq1SpQoYWnr7u6unj17mlIwAAAAAADpUYoDtiS98MILGjx4sFm1AAAAAACQYaXoGmwAAAAAAGAtVWewIyMjdfToUV2/fl1RUVGJtmnfvn1qNgEAAAAAQIaQ4oC9ZMkSzZgxQyEhIYnONwxDdnZ2BGwAAAAAQKaQoiHimzdv1scff6wCBQpo5MiRMgxDTZo00dChQ1WvXj0ZhqFmzZpp4sSJZtcLAAAAAEC6lKKA/b///U958uTR8uXL1atXL0lS2bJl1b9/f3311Vf67LPPtG3bNhUqVMjMWgEAAAAASLdSFLBPnTqlxo0by8XFxTItLi7O8v9t2rRRrVq1NGfOnNRXCAAAAABABpCigB0TEyN3d3fL66xZs+revXtWbby8vHTixInUVQcAAAAAQAaRooDt4eGh69evW14XKlRIJ0+etGpz5coVOTg4pK46AAAAAAAyiBQF7IoVK1qdna5Xr54OHTqkefPm6fTp01q2bJm2bNmiihUrmlYoAAAAAADpWYoCdosWLRQVFaVLly5JkgYMGKACBQpo+vTpatu2rcaNGydXV1e9++67phYLAAAAAEB6laLnYDdt2lRNmza1vHZ3d9ePP/6oH374QRcvXlThwoXVrl075c+f37RCAQAA0poRFyc7+xSdf8iQMtv+AkBaS1HATkzOnDnVt29fs1YHAADwzNnZ2+vw3HkKvXLV1qWkObdCBeXznwG2LgMAnispCtg9evRQx44d1b59+8e2+emnn7Rq1SotWrQopbUBAAA8c6FXrure+fO2LgMAkAGlaEzQgQMHLNdfP86VK1f022+/pagoAAAAAAAymjS76CY8PFxZspg2Ah0AAAAAgHQtyQn4ypUrVq9DQkISTJOk2NhYBQUFadOmTSpcuHDqKwQAAAAAIANIcsBu3Lix7OzsJEl2dnZatGjRE6+vNgxDI0aMSH2FAAAAAABkAEkO2O3bt5ednZ0Mw9CPP/6osmXLqly5cgna2dvbK2fOnKpVq5bq169varEAAAAAAKRXSQ7Yn3zyieX/Dxw4oI4dO6pHjx5pUhQAAAAAABlNiu5Ctn37drPrAAAAAAAgQ0u3t/k+fvy4Zs2apUOHDikyMlJFixZV586dLWfN/f39deDAgQTL+fr6asGCBc+6XAAAAABAJpfigL1nzx4tXLhQR48eVUhIiOLi4hK0sbOz04kTJ5K97l27dumNN95Q+fLlNXDgQLm6uurChQsKCgqyalegQAENGzbMapqHh0eytwcAAAAAQGqlKGBv2rRJQ4cOVVxcnAoVKqSSJUvKwcHBlIJCQ0M1cuRINWzYUDNnzpS9/eMf1Z09e3a1a9fOlO0CAAAAAJAaKQrYc+bMkbOzs7744gvVrl3b1IICAgJ08+ZNDR06VPb29goLC1PWrFkfG7RjYmIUGRmpbNmymVoHAAAAAADJ8fjTw09w9uxZtW7d2vRwLUl79+6Vm5ubrl27pubNm6tKlSqqVq2aPvzwQ0VGRlq1PXfunHx8fFS1alXVrVtX06dPV3R0tOk1AQAAAADwNCk6g50rVy5lzZrV7FokPQjNsbGxGjhwoF555RW98847OnDggBYvXqyQkBB9/vnnkqSiRYuqZs2a8vT0VFhYmDZt2qS5c+fq3Llzmj59eqpqiIyMVGxsrAl7k3R2dnZycXF5ptsEnoXw8HAZhmHrMpAG6Lcyh8z0Gc6sv9OZ5WecWX++eP5lls+wrbm6uiapXYoCdvPmzbV3717FxMQoSxZzb0QeFham8PBwde3aVWPGjJEkNWvWTFFRUVq+fLnefPNNvfDCC5o4caLVcu3bt9cHH3ygFStWqFevXvLx8UlxDceOHUvNLqSIi4uLypcv/8y3C6S1s2fPKjw83NZlIA3Qb2UOmekznFl/pzPLzziz/nzx/Mssn2Fbq1atWpLapSgdDxs2TL1799bQoUM1evRoFSpUKCWrSVT8mXE/Pz+r6W3atNHy5ct1+PBhvfDCC4ku+/rrr2vFihXas2dPqgK2t7e3Tc5gA8+jEiVK8K3qc4p+K3PITJ/hzPo7nVl+xpn154vnX2b5DGcUKQrYbdq0UUxMjP78809t3bpVOXLkkJubW4J2dnZ22rp1a7LW7eHhodOnTytPnjxW093d3SVJwcHBj122YMGCT22TFM7OzqlaHsD/YTgekLHxGX7+8TMGMjY+w+lLim5yZhiGHBwcVLBgQRUsWFDZsmWTYRgJ/iX2bOynqVChgiTp2rVrVtOvX78u6f+CdmIuXrz41DYAAAAAAKSFFJ3B3r59u9l1WLRs2VJfffWVVq5caXWX8pUrVypLliyqUaOGQkND5eTkJCcnJ8t8wzA0d+5cSZKvr2+a1QcAAAAAQGLMvUOZCcqXL6+XX35Zq1atUmxsrF588UUdOHBAGzdu1IABA5Q/f37t379f77zzjlq3bq1ixYopMjJSW7Zs0aFDh9SlSxfLWXAAAAAAAJ6VVAfsf/75R2fOnFFYWJjat29vQknS+PHjVahQIa1evVpbt25VoUKFNHr0aPXq1UuSVKhQIVWrVk1btmzRzZs3ZW9vr5IlS2r8+PHq0qWLKTUAAAAAAJAcKQ7YR44c0QcffKC///7bMi0+YP/222/q27evPv/8czVp0iTZ63Z0dNTgwYM1ePDgROcXLVpUM2bMSFHdAAAAAACkhRTd5Oz06dPq2bOnLl26pF69eql+/fpW86tXr67cuXNr48aNphQJAAAAAEB6l6KAPWvWLEnS6tWrNXLkSFWsWNFqvp2dnXx8fHT06NHUVwgAAAAAQAaQooB94MABNW/eXMWLF39sm4IFC+rGjRspLgwAAAAAgIwkRQH7/v37T33WdGRkZIqegw0AAAAAQEaUooBdsGBBq5ubJebEiRMqWrRoiooCAAAAACCjSVHAbtiwoXbv3q09e/YkOn/9+vU6fPiwXnrppVQVBwAAAABARpGix3S98cYb2rRpk/r376/27dvr5s2bkqQlS5bo8OHDWrdunQoXLqzXX3/d1GIBAAAAAEivUhSw3d3d9d133+ndd9/VypUrLdM//vhjSVLlypU1depUZc+e3ZwqAQAAAABI51IUsCWpaNGiWrZsmU6ePKnDhw8rODhYbm5uqlSpkipVqmRmjQAAAAAApHspDtjxypUrp3LlyplRCwAAAAAAGVaKbnIWEhKiv/76S+Hh4YnODwsL019//aXQ0NBUFQcAAAAAQEaRooA9Z84cvfrqq4qNjU10flxcnF599VXNnTs3VcUBAAAAAJBRpChg//rrr6pbt67c3NwSne/m5qZ69epp586dqSoOAAAAAICMIkUB+8qVKypevPgT2xQtWlRXrlxJUVEAAAAAAGQ0KQrYdnZ2ioqKemKbqKgoxcXFpagoAAAAAAAymhQF7JIlS+rXX3+VYRiJzo+Li9POnTtVokSJVBUHAAAAAEBGkaKA7efnp3Pnzum9995TSEiI1byQkBC99957unDhgtq2bWtKkQAAAAAApHcpeg52t27dtGnTJq1Zs0bbtm1TxYoV5eHhoevXr+vo0aO6d++eXnzxRXXr1s3segEAAAAASJdSFLAdHR317bffavr06VqxYoV2795tmefm5qY+ffrorbfekqOjo2mFAgAAAACQnqUoYEuSs7OzRo4cqeHDh+vMmTMKCQlRjhw5VKJECTk4OJhZIwAAAAAA6V6KAnaTJk1Uv359ffjhh3JwcFCZMmXMrgtABpfTJbuMuDjZ2afoVg8ZUmbbXwAAAFhLUcC+c+eO3NzczK4FwHPE1clFdvb2Ojx3nkKvXLV1OWnOrVBB+fxngK3LAAAAgA2lKGB7eXnp3LlzJpcC4HkUeuWq7p0/b+syAAAAgDSXorGM/fr1088//6x9+/aZXQ8AAAAAABlSis5g37t3T3Xr1lWfPn3UpEkTVaxYUXnz5pWdnV2Ctu3bt09tjQAAAAAApHspCtijRo2SnZ2dDMPQ5s2btXnzZkmyCtiGYcjOzo6ADQAAAADIFFIUsCdNmmR2HQAAAAAAZGgpCtgdOnQwuw4AAAAAADI0HtgKAAAAAIAJUnQGO96WLVsUGBioM2fOKCIiQlu2bJEk/fvvv9q+fbvatm2r/Pnzm1IoAAAAAADpWYoCdlxcnIYNG6ZNmzZJkrJmzaqIiAjL/Jw5c2r69OmKi4vTgAEDzKkUAAAAAIB0LEVDxL/99ltt3LhRXbp00W+//abevXtbzc+bN6+qVaumX375xYwaAQAAAABI91IUsNesWaOKFStq3LhxcnNzS/T518WLF9elS5dSXSAAAAAAABlBigL2+fPnVb169Se2yZUrl+7evZuS1QMAAAAAkOGkKGBnzZpVISEhT2xz5coV5ciRI0VFAQAAAACQ0aQoYJcrV067du1SZGRkovPv3r2rX3/9VZUrV05VcQAAAAAAZBQpCtj+/v4KCgrSkCFDFBQUZDXvwoULGjx4sEJCQuTv729KkQAAAAAApHcpekzXSy+9pH79+unrr79Wo0aN5OLiIkmqXbu27t69K8MwNHDgQNWuXdvUYgEAAAAASK9SFLAl6Z133lGtWrX03Xff6ciRI4qKilJcXJzq1asnf39/1atXz8w6AQAAAABI11IcsCWpbt26qlu3rlm1AAAAAACQYSXrGuw//vhDPXr0UJUqVVS1alW9/vrrOnLkSFrVBgAAAABAhpHkM9inTp1Sr169rO4cvnfvXv3xxx/64YcfVKZMmTQpEAAAAABgLadLdhlxcbKzT9F9qzOs9L7PSQ7YX331lSIjI/XGG29Y7g6+ZMkSffHFF/r66681efLkNCsSAAAAAPB/XJ1cZGdvr8Nz5yn0ylVbl/NMuBUqKJ//DLB1GU+U5IB98OBBVatWTW+//bZl2ptvvqkDBw7ot99+S4vaAAAAAABPEHrlqu6dP2/rMvD/Jfnc+s2bN1W5cuUE0ytVqqSbN2+aWhQAAAAAABlNkgN2TEyMXF1dE0x3dXVVTEyMqUUBAAAAAJDRpN+rwwEAAAAAyECS9RzsgIAA/fnnn1bTLly4IEnq169fgvZ2dnb66quvUlEeAAAAAAAZQ7IC9vnz53X+MRfQ//rrrwmm2dnZpawqAAAAAAAymCQH7G3btqVlHQAAAAAAZGhJDtiFCxdOyzoAAAAAAMjQuMkZAAAAAAAmIGADAAAAAGACAjYAAAAAACYgYAMAAAAAYAICNgAAAAAAJiBgAwAAAABgAgI2AAAAAAAmIGADAAAAAGACAjYAAAAAACYgYAMAAAAAYAICNgAAAAAAJiBgAwAAAABgAgI2AAAAAAAmIGADAAAAAGCCdBuwjx8/rjfeeEM1atRQ5cqV5efnp0WLFlm1OXTokF599VVVrlxZdevW1YQJE3T//n0bVQwAAAAAyMyy2LqAxOzatUtvvPGGypcvr4EDB8rV1VUXLlxQUFCQpc3JkyfVq1cvlSpVSqNGjVJQUJC++eYbnTt3TvPnz7dh9QAAAACAzCjdBezQ0FCNHDlSDRs21MyZM2Vvn/hJ9s8//1w5cuTQ4sWL5ebmJkkqUqSIxowZo127dsnX1/dZlg0AAAAAyOTS3RDxgIAA3bx5U0OHDpW9vb3CwsIUFxdn1SY0NFR79uxR27ZtLeFaktq1aydXV1dt2LDhWZcNAAAAAMjk0l3A3rt3r9zc3HTt2jU1b95cVapUUbVq1fThhx8qMjJSknTq1CnFxMTI29vbalknJyeVK1dOJ0+etEXpAAAAAIBMLN0NET937pxiY2M1cOBAvfLKK3rnnXd04MABLV68WCEhIfr8889148YNSZKHh0eC5fPly6eDBw+mqobIyEjFxsamah3JZWdnJxcXl2e6TQDmCw8Pl2EYti7jmaDfyhwiIiIy1e901qxZbV3GM5dZ+i36LOD5YYt+y9XVNUnt0l3ADgsLU3h4uLp27aoxY8ZIkpo1a6aoqCgtX75cb775piIiIiQ9OGP9KGdnZ8v8lDp27Fiqlk8JFxcXlS9f/plvF4C5zp49q/DwcFuX8UzQbz3fsmTLobi4uEwZODObzNJv0WcBzw9b9FvVqlVLUrt0F7Dj/5D7+flZTW/Tpo2WL1+uw4cPW9pERUUlWD4yMjLVBwPe3t42OYMNIOMrUaJEpjgTJNFvPe+yOLvK3t5e83Ys0pXga7Yu55moVLicXq7m9/SGz5nM0m/RZwHPj/Tcb6W7gO3h4aHTp08rT548VtPd3d0lScHBwSpatKgk6fr16wmWv3HjRqJDx5PD2dk5VcsDyLwYfojnzZXgazp/65Kty3gmCuZM3fFDRkW/BSCjSc/9Vrq7yVmFChUkSdeuWX9bHh+m3d3d5enpqSxZsiQYyh0VFaWTJ0+qbNmyz6ZYAAAAAAD+v3QXsFu2bClJWrlypdX0lStXKkuWLKpRo4ayZ8+u2rVra+3atQoNDbW0+emnnxQWFqYWLVo805oBAAAAAEh3Q8TLly+vl19+WatWrVJsbKxefPFFHThwQBs3btSAAQOUP39+SdLQoUPVtWtX+fv7q3PnzgoKCtLChQvl6+ur+vXr23gvAAAAAACZTboL2JI0fvx4FSpUSKtXr9bWrVtVqFAhjR49Wr169bK0qVChghYuXKgpU6Zo0qRJypYtm1555RUNGzbMdoUDAAAAADKtdBmwHR0dNXjwYA0ePPiJ7apXr65ly5Y9o6oAAAAAAHi8dHcNNgAAAAAAGREBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBGwAAAAAAExCwAQAAAAAwAQEbAAAAAAATELABAAAAADABARsAAAAAABNksXUBj9q/f7969OiR6Lzly5fLx8dHkuTv768DBw4kaOPr66sFCxakZYkAAAAAACSQ7gJ2PH9/f1WsWNFqWrFixaxeFyhQQMOGDbOa5uHhkea1AQAAAADwqHQbsKtXr64WLVo8sU327NnVrl27Z1QRAAAAAACPl66vwQ4NDVVMTMwT28TExOj+/fvPqCIAAAAAABKXbs9gjx49WmFhYXJwcFC1atU0YsSIBEPGz507Jx8fH0VHRytv3rzq1KmTBg0aJEdHRxtVDQAAAADIrNJdwHZ0dFTz5s1Vv3595c6dW//++68WLFigbt26admyZSpfvrwkqWjRoqpZs6Y8PT0VFhamTZs2ae7cuTp37pymT5+eqhoiIyMVGxtrwt4knZ2dnVxcXJ7pNgGYLzw8XIZh2LqMZ4J+C3g+ZJZ+iz4LeH7Yot9ydXVNUrt0F7CrVq2qqlWrWl43adJEzZs3V9u2bTV16lTLHcInTpxotVz79u31wQcfaMWKFerVq5flbuMpcezYsRQvm1IuLi6WLw8AZFxnz55VeHi4rct4Jui3gOdDZum36LOA54ct+q1q1aolqV26C9iJKV68uJo0aaLNmzcrNjZWDg4OibZ7/fXXtWLFCu3ZsydVAdvb29smZ7ABZHwlSpTIFGeCJPot4HmRWfot+izg+ZGe+60MEbClB4/kio6OVnh4uNzc3BJtU7BgQUlScHBwqrbl7OycquUBZF4MPwSQ0dBvAcho0nO/la7vIv6wS5cuydnZ+Ylj3y9evChJcnd3f1ZlAQAAAAAgKR0G7Nu3byeY9tdff2n79u2qW7eu7O3tFRoaqqioKKs2hmFo7ty5kiRfX99nUisAAAAAAPHS3RDxt99+W1mzZlWVKlWUJ08e/fPPP1qxYoWyZs2q4cOHS5KOHz+ud955R61bt1axYsUUGRmpLVu26NChQ+rSpYsqVKhg470AAAAAAGQ26S5gv/TSSwoICNC3336r0NBQ5c6dW02bNtXgwYNVvHhxSVKhQoVUrVo1bdmyRTdv3pS9vb1Kliyp8ePHq0uXLjbeAwAAAABAZpTuAnaPHj3Uo0ePJ7YpWrSoZsyY8YwqAgAAAADg6dLdNdgAAAAAAGREBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABARsAAAAAABMQMAGAAAAAMAEBGwAAAAAAExAwAYAAAAAwAQEbAAAAAAATEDABgAAAADABFlsXcCj9u/frx49eiQ6b/ny5fLx8bG8PnTokD777DOdOHFCbm5uatmypYYOHaps2bI9o2oBAAAAAHgg3QXseP7+/qpYsaLVtGLFiln+/+TJk+rVq5dKlSqlUaNGKSgoSN98843OnTun+fPnP+tyAQAAAACZXLoN2NWrV1eLFi0eO//zzz9Xjhw5tHjxYrm5uUmSihQpojFjxmjXrl3y9fV9VqUCAAAAAJC+r8EODQ1VTExMotP37Nmjtm3bWsK1JLVr106urq7asGHDsywTAAAAAID0ewZ79OjRCgsLk4ODg6pVq6YRI0ZYhoyfOnVKMTEx8vb2tlrGyclJ5cqV08mTJ21RMgAAAAAgE0t3AdvR0VHNmzdX/fr1lTt3bv37779asGCBunXrpmXLlql8+fK6ceOGJMnDwyPB8vny5dPBgwdTVUNkZKRiY2NTtY7ksrOzk4uLyzPdJgDzhYeHyzAMW5fxTNBvAc+HzNJv0WcBzw9b9Fuurq5JamdnZIAe9fz582rbtq2qV6+uBQsW6Mcff9TIkSP1ww8/qFKlSlZtR4wYoe3bt+v333+3UbUAAAAAgMwoXV+DHa948eJq0qSJ9u/fr9jYWGXNmlWSFBUVlaBtZGSkZT4AAAAAAM9KhgjYklSgQAFFR0crPDxc+fLlkyRdv349QbsbN24kOnQcAAAAAIC0lGEC9qVLl+Ts7CxXV1d5enoqS5YsOnbsmFWbqKgonTx5UmXLlrVRlQAAAACAzCrdBezbt28nmPbXX39p+/btqlu3ruzt7ZU9e3bVrl1ba9euVWhoqKXdTz/9pLCwsCc+PxsAAAAAgLSQ7m5y1qNHD2XNmlVVqlRRnjx59M8//2jFihXKkiWLli9frlKlSkmSjh8/rq5du6p06dLq3LmzgoKCtHDhQr344otasGCBjfcCAAAAAJDZpLuAvWjRIgUEBOjChQsKDQ1V7ty5Vbt2bQ0ePFjFixe3avv7779rypQpOnHihLJly6aWLVtq2LBhcnNzs1H1AAAAAIDMKt0FbAAAAAAAMqJ0dw02AAAAAAAZEQEbAAAAAAATELABAAAAADABARsAAAAAABMQsAEAAAAAMAEBG8hE9u/fLy8vL23cuNHWpQCZVuPGjTVq1KhkLcNnN3GjRo1SlSpVbF0GgOfArFmz5OXlZesy8BwgYCPdWL16tby8vHT06NFkLRceHq5Zs2Zp//79aVRZ0ly6dEleXl5asGDBE9s1btxYXl5e6tWrV6LzV6xYIS8vr2S/F/v379fgwYNVt25deXt7q3bt2nrjjTe0efPm5OxGsu3YsUOzZs1K020AthLfLz38r3bt2vL399eOHTtsXV66s3fvXo0ePVrNmzdX5cqV1aRJE73//vu6fv16stYTGRmpb7/9Vp06dVK1atVUsWJFNW/eXB999JHOnj2bRtU/8OWXX2rr1q1pug3Alp52vJXU45mk8vf3l5eXl5o1a5bo/N27d1v618d9ibhkyRJ5eXmpU6dOKaohNjZWvr6+8vLyou9Gmsti6wKA1AoPD9fs2bM1ePBg1axZ09blJImzs7P279+vGzduKF++fFbzAgIC5OzsrMjIyCSvb+bMmZozZ45eeOEFdenSRYUKFdLdu3e1Y8cODRkyRFOmTFGbNm3M3g1JDwL2kiVLNGTIkDRZP5AevPnmmypSpIgMw9CtW7e0Zs0a9e/fX19++aUaNWqUrHVt3LhRdnZ2aVSpbX322WcKDg5WixYt9MILL+jixYv67rvv9Msvv+jHH39M0N8l5vbt2+rbt6+OHz+uRo0ayc/PT66urjp79qzWr1+vFStW6NixY2m2D/PmzVPz5s310ksvpdk2gMzG2dlZ58+f15EjR1SpUiWreUk57gkICFDhwoV15MgRnT9/XsWLF0/W9vft26cbN26ocOHCCggIUIMGDVK0H0BSELCBxwgLC5Orq2uarLtq1ao6evSo1q9fr549e1qmBwUF6ffff1fTpk21adOmJK1r48aNmjNnjpo3b66pU6fK0dHRMq9v37769ddfFRMTY/o+AJlJ/fr1VbFiRcvrV155RXXr1lVgYGCyA7aTk5PZ5aUbo0ePVrVq1WRv/38D5OrVq6fu3bvru+++09ChQ5O0jpMnT2rmzJlq3ry51by3335b06ZNM71uAGmrWLFiiomJUWBgoFXAjoyM1JYtW9SwYcPHHvdcvHhRf/zxh2bPnq2xY8cqICBAgwcPTtb2165dqwoVKqh9+/aaNm1amh7jAQwRR7oVf23dtWvXNHDgQFWpUkW1atXSp59+qtjYWEkPhjHVrl1bkjR79mzLEKOHhyz/+++/evPNN1WjRg1VrFhRHTt21LZt26y2FT9c6sCBAxo3bpxq165t+XbT399ffn5++ueff+Tv76/KlSurXr16+vrrr1O8b87OzmrWrJkCAwOtpgcGBipHjhzy9fVN8rpmzJihXLlyaeLEiVbhOl69evUSBIC4uDjNnTvXEhp69uyp8+fPW7X5/fff9eabb6phw4by9vZWgwYNNHHiREVERFjajBo1SkuWLJEkqyG0wPMuR44ccnZ2VpYs//c99YIFC9S1a1fVrFlTlSpVUseOHRMd7pjYNdj37t3TxIkT1bhxY3l7e6t+/foaMWKEbt++bdUuKZ/dx13j7e/vL39/f8vrqKgozZgxQx07dlS1atXk4+Oj1157Tfv27bNa7uHhosuXL9dLL70kb29vvfzyyzpy5IhV2xdffNEqXMdPy5Url86cOZPYW2nlzz//1C+//KJXXnklQbiWHnw5MXLkyATTn/R3Il5Sfj5eXl4KCwvTmjVrLP1Zcq+XBzKLVatWqUePHqpdu7a8vb3VqlUrff/9949t7+fnp/Xr1ysuLs4ybfv27YqIiFCLFi0eu1xAQIBy5sypBg0aqHnz5goICEhWnREREdqyZYtatWqlli1bKiIiIsFx4ON4eXnpo48+0tq1a9W8eXPLceRvv/1m1e7y5csaN26cmjdvrkqVKqlmzZp68803denSpQTrfFp/n9S+GekXZ7CRrsXGxqpPnz6qVKmSRowYob179+qbb75R0aJF9dprr8nd3V3jxo3TuHHj1LRpUzVt2lSSLCHv9OnTevXVV5U/f37169dPrq6u2rBhgwYNGqRZs2ZZ2scbP3683N3dNWjQIIWFhVmmBwcHq2/fvmratKlatmypTZs2acqUKfL09EzxMCM/Pz/17t1bFy5cULFixSQ9CNjNmze3Omh/knPnzunMmTN6+eWX5ebmluRtf/3117Kzs1Pv3r0VGhqq+fPna/jw4frhhx8sbTZu3KiIiAi9+uqrypUrl44cOaLvvvtOQUFBmjlzpiSpS5cuun79unbv3q3JkycnY++BjCU0NNRy8HPr1i0tXrxYYWFhatu2raXNokWL1LhxY7Vp00bR0dFat26d3nrrLc2bN08NGzZ87Lrv37+vbt266d9//9XLL7+s8uXL686dO9q+fbuuXbsmd3d3S9ukfHaTs08//PCD/Pz81KlTJ92/f18rV65U37599cMPP6hcuXJW7QMDA3X//n116dJFdnZ2mj9/voYMGaKtW7cm+uXew/t3//595c6d+6k1bd++XZLUrl27JO/H0/5OxEvKz2fy5MkaM2aMKlWqpM6dO0uSpX8GYG3p0qUqU6aMGjdurCxZsujnn3/W+PHjZRiGunXrlqC9n5+f5Z458SdHAgMDVatWLeXJk+ex2wkICFDTpk3l5OQkPz8/LV26NNGh5o+zfft2hYWFqXXr1sqXL59q1KihgICAJF8699tvv2n9+vXy9/eXk5OTli5dauknPT09JUlHjx7VH3/8odatW6tAgQK6fPmyli5dqh49emjdunVycXGRlLT+Prl9M9IhA0gnVq1aZXh6ehpHjhwxDMMwRo4caXh6ehqzZ8+2ate+fXujQ4cOlte3bt0yPD09jZkzZyZYZ8+ePQ0/Pz8jMjLSMi0uLs7o0qWL0axZswTbfvXVV42YmBirdXTv3t3w9PQ01qxZY5kWGRlp1K1b1xgyZIhl2sWLFw1PT09j/vz5T9zPRo0aGf379zdiYmKMunXrGnPmzDEMwzD++ecfw9PT0zhw4ECC9+Jxtm7danh6ehoLFy58Yrt4+/btMzw9PY2WLVtavSf/+9//DE9PT+PUqVOWaeHh4QmWnzdvnuHl5WVcvnzZMm38+PGGp6dnkrYPZDTxn8VH/3l7exurV6+2avvoZyYqKsrw8/MzevToYTW9UaNGxsiRIy2vZ8yYYXh6ehqbN29OsP24uDjDMJL32X10/fG6d+9udO/e3fI6JibGal2GYRjBwcFGnTp1jNGjR1umxfdtNWrUMO7evWuZHt//bN++PcG2HjZnzhzD09PT2LNnzxPbGYZhDBo0yPD09DSCg4Of2tYwkv53wjCS/vPx8fFJ9P0DnhdPO8ZI6vFMYscJvXv3Npo0aWI1rXv37kbr1q0NwzCMjh07Gu+9955hGA/6mwoVKhhr1qyx9HEbNmywWvbo0aOGp6ensXv3bsMwHvSJ9evXNyZMmJC0nTUMY8CAAUbXrl0tr5cvX26UL1/euHXrllW7mTNnJjieie/zjx49apl2+fJlo2LFisagQYOe+F788ccfCY4fk9LfJ7VvRvrFEHGke6+++qrV62rVqiU65OZRd+/e1b59+9SyZUvL2afbt2/rzp078vX11blz53Tt2jWrZTp37iwHB4cE63J1dbU6o+Lk5KSKFSvq4sWLKdwrycHBQS1atNC6deskPbg+qGDBgqpevXqS1xEaGipJypYtW7K23bFjR6vrQOO3+fD+ZM2a1fL/YWFhun37tqpUqSLDMHTixIlkbQ/I6MaOHauFCxdq4cKF+uyzz1SzZk2NGTPG6i79D39mgoODFRISomrVqj3187J582aVLVs2wYgaSQluhpaUz25SOTg4WNYVFxenu3fvKiYmRt7e3onW3KpVK+XMmTNZ2/7tt980Z84ctWzZ0nLG6klS2qcl5e9ESn8+ABL38GcqJCREt2/fVo0aNXTx4kWFhIQkukybNm20ZcsWRUVFadOmTXJwcHjiDQUDAgKUN29ey01s7ezs1KpVK61fvz7BZSCJuXPnjnbt2iU/Pz/LtGbNmsnOzk4bNmxI0n5WqVJF3t7elteFChVSkyZNtGvXLksND78X0dHRunPnjooVK6YcOXJY9TFJ6e+T2zcj/WGIONI1Z2dnq+GRkpQzZ04FBwc/ddkLFy7IMAzNmDFDM2bMSLTNrVu3lD9/fsvrIkWKJNquQIECCQ50c+bMqVOnTj21jidp06aNFi9erL/++kuBgYFq1apVoncXjoiISPDHKl++fJZh4ffv30/WdgsVKmT1OkeOHJIeXBcU78qVK5o5c6a2b9+e4P2OPwgGMotKlSpZ3eTMz89P7du310cffaSGDRvKyclJP//8s+bOnauTJ08qKirK0vZpdwy/cOHCYx9f86ikfHaTY82aNfrmm2909uxZRUdHW6Yn1hcWLFjQ6nV82H7ctv/9918NHjxYZcqU0YQJE6zm3b1712p7WbNmVfbs2a36tPh9e5qk/p1I6c8HQOIOHjyoWbNm6fDhwwoPD7eaFxISouzZsydYplWrVvr000+1c+dOrV27Vg0bNnzsJW6xsbFat26datasafWFWaVKlfTNN99o79698vX1VVRUVILPu7u7uxwcHLR+/XpFR0erXLlyVverqFSpkgICAhIdyv6oxO5Y/sILLyg8PFy3b99Wvnz5FBERoXnz5mn16tW6du2aDMOwei/iJbW/T07fjPSHgI10LbGzyUkVfxON3r17q169eom2efTaOmdnZ9PreJLKlSurWLFi+u9//6tLly499nqg9evXa/To0VbTTp06pZIlS0qS/v7772Rt99GbEMWL/4MQGxur119/3XLtecmSJeXq6qpr165p1KhRVjcoATIje3t71axZU4sWLdL58+cVHBys//znP3rxxRf14YcfKl++fHJ0dNSqVasS3MwwtdtNzMMHc48TGxtr1Zf99NNPGjVqlF566SX16dNHefLkkYODg+bNm5foWenH9YOJbfvq1avq06eP3Nzc9NVXXyU4gB4yZIgOHDhged2hQwd98sknVn1aUkfzJKV//v3335/JzwfILC5cuKBevXqpZMmSGjVqlAoWLChHR0ft2LFD33777WOPEzw8PFSjRg0tXLhQhw4dsrop7aPiH621bt06y2i/hwUEBMjX11d//PGHevToYTVv27ZtKlKkiOWGaI+Ocol38eJFFS1aNKm7/Vgff/yxVq9erZ49e8rHx0fZs2eXnZ2dhg4dmqT++WHJ7ZuR/hCwkeE97uxDfIfp6OioOnXqPMuSkqV169aaO3euSpUq9dgbV/j6+mrhwoUJppcoUUIlSpTQtm3bdP/+/WQPq3ycv//+W+fOndOnn36q9u3bW6bv3r07QVvO/iCzih8aGBYWpk2bNsnZ2VkLFiywGsK9atWqp66nWLFiOn36tGl15cyZM9GzyleuXLE6kNy0aZOKFi2q2bNnW32O429imFJ37txR7969FRUVpe+//14eHh4J2owcOdKqxvg2jRo10rx587R27dpkXS7zNKn5+QBIaPv27YqKitLcuXOtRtbs37//qcv6+flpzJgxypEjh+rXr//YdgEBAcqTJ4/Gjh2bYN6WLVu0ZcsWjR8/XmXLlk1wjJQvXz7L4726d++uF1980Wp+XFycRowYoYCAAA0cOPCJ9T76pAbpwU1mXVxcLKNnNm3apPbt21s9dSAyMjLB6MOk9Pdp1Tfj2SFgI8OLvzPjoweUefLkUY0aNbR8+XJ17949wUHe7du3EwwrtIVOnTrJwcHhiXfD9PDwSPQgVZLefPNNDR06VGPGjNFnn32W4A7ku3btUnR0dLKe1Rt/luzhb10Nw9CiRYsStH34/U/qkE4go4uOjtbu3bvl6OioUqVKycHBQXZ2dlbXBF66dClJj4Jp1qyZ5syZoy1btiS4Ls8wjGR/iVW0aFEdPHhQUVFRljD5888/6+rVq1YBO/7M78Pb+PPPP3X48OEEQ9GTKiwsTP3799e1a9e0aNEivfDCC4m2e/h6xodVqVJF9erV0w8//KD69esnuDYzKipK06ZNS/RRXU+SnJ+Pq6triofcA5nFw/1HvJCQkCR9adWiRQsFBQWpRIkSVl94PSwiIkKbN29WixYtEn2El4eHhwIDA7V9+3a1atUq0RMp8Wev+/btm+ASF0lauXJlkgL2H3/8oePHj6tChQqSHozQ2bZtm+rVq2d5HxIbSbN48eIE14knpb9Pi74ZzxYBGxle1qxZVbp0aW3YsEEvvPCCcuXKpTJlysjT01MffvihXnvtNbVp00adO3dW0aJFdfPmTR0+fFhBQUFau3at6fXs3btXkZGRCaa/9NJLlsc5PKxw4cIaMmRIirfXqlUrnTp1Sl9++aVOnDghPz8/FSpUSHfv3tWvv/6qvXv3aurUqclaZ8mSJVWsWDF9+umnunbtmtzc3LRp06ZEDzrj/+BMmDBBvr6+cnBwUOvWrVO8P0B6tHPnTstznG/fvq2AgACdO3dO/fv3l5ubmxo0aKCFCxeqb9++8vPz061bt/T999+rWLFiT71XQ58+fbRp0ya99dZbevnll1WhQgUFBwdr+/btlrMzydGpUydt2rRJffv2VcuWLXXhwgUFBAQkuCSmYcOG2rx5swYNGqSGDRvq0qVLWrZsmUqXLm31mMLkGD58uI4cOaKXX35Z//77r/7991/LvGzZsj3xZkbxJk+erN69e2vw4MFq1KiRateuLRcXF50/f17r16/X9evXkx2wk/PzqVChgvbu3auFCxfKw8NDRYoUUeXKlZO1PSAjWLVqlX799dcE05s0aSLpycczdevWlaOjo9544w117dpV9+/f1w8//KA8efLoxo0bT9xu9uzZn3rcs337dt2/f1+NGzdOdL6Pj4/c3d21du1atWrVKtE2AQEBKleuXKLhWpIaN26sjz/+2Co8J8bT01N9+vSxekyXJKt9aNiwoX766Se5ubmpdOnSOnz4sPbs2aNcuXJZrSsp/X1a9M14tgjYeC5MmDBBH3/8sSZNmqTo6GgNHjxYnp6eKl26tFatWqXZs2drzZo1unv3rtzd3VW+fHkNGjQoTWr59ddfE/2DVbhw4UQDthmGDh2qWrVqafHixVq6dKmCg4OVI0cOVa5cWV988YXlj2VSOTo66ssvv9SECRM0b948OTs7q2nTpurWrVuC59M2a9ZM/v7+WrdundauXSvDMAjYeO48PDTP2dlZJUuW1Lhx49S1a1dJUu3atfXf//5XX3/9tSZOnKgiRYpo+PDhunz58lMDdrZs2bRkyRLNmjVLW7Zs0Zo1a5QnTx7Vrl3b6iaMSVWvXj2NGjVKCxcu1MSJE+Xt7a0vv/xSn376qVW7jh076ubNm1q+fLl27dql0qVL67PPPtPGjRutro9Ojr/++kvSgwP3R89kFS5cOEkB293dXcuWLdP333+v9evXa9q0aYqOjlbhwoXVuHHjBNdaJkVyfj6jRo3S2LFjNX36dEVERKhDhw4EbDyX4oPio2rUqCHpyccz7dq108yZMzV9+nR9+umnyps3r1599VW5u7vrvffeS3Vta9eulbOzs+rWrZvofHt7ezVs2FABAQG6c+eOcufObTX/+PHjOnPmzBPPTjdq1Egff/yx1q5d+8SA/eKLL8rHx0dz5szRlStXVLp0aU2aNMnqy8/3339f9vb2CggIUGRkpKpWrWr5Uu9hSenv06JvxrNlZyT3ynsAAAAAeM55eXmpW7duiV4HDjwOz8EGAAAAAMAEBGwAAAAAAExAwAYAAAAAwARcgw0AAAAAgAk4gw0AAAAAgAkI2AAAAAAAmICADQAAAACACQjYAAAAAACYgIANAAAAAIAJCNgAACDZvLy8NGvWrGQvd+nSJXl5eWn16tVpUBUAALZFwAYAIANbvXq1vLy85OXlpd9//z3BfMMw1KBBA3l5eWnAgAE2qBAAgMyDgA0AwHPA2dlZgYGBCaYfOHBAQUFBcnJyskFVAABkLgRsAACeAw0aNNDGjRsVExNjNT0wMFAVKlRQvnz5bFQZAACZBwEbAIDnQOvWrXX37l3t3r3bMi0qKkqbNm1SmzZtErQPCwvTJ598ogYNGsjb21vNmzfXggULZBiGVbuoqChNnDhRtWrVUpUqVfTGG28oKCgo0RquXbum0aNHq06dOvL29lbr1q21cuVKc3cUAIB0LIutCwAAAKlXuHBh+fj4aN26dWrQoIEkaefOnQoJCVGrVq20ePFiS1vDMPSf//xH+/fv1yuvvKJy5crp119/1eTJk3Xt2jW99957lrbvv/++1q5dKz8/P1WtWlX79u1T//79E2z/5s2b6ty5s+zs7NStWze5u7tr586dev/99xUaGqpevXql+XsAAICtcQYbAIDnRJs2bbR161ZFRERIkgICAvTiiy8qf/78Vu22bdumffv26a233tKECRPUrVs3ffnll2revLkWLVqkCxcuSJL++usvrV27Vq+99pqmTp2qbt26adasWSpTpkyCbU+bNk2xsbFas2aNBg0apFdffVVz585V69atNXv2bEtNAAA8zwjYAAA8J1q2bKnIyEj9/PPPCg0N1S+//JLo8PCdO3fKwcFB/v7+VtN79+4twzC0c+dOSdKOHTskKUG7nj17Wr02DEObN29W48aNZRiGbt++bfnn6+urkJAQHT9+3MxdBQAgXWKIOAAAzwl3d3fVrl1bgYGBioiIUGxsrJo3b56g3eXLl+Xh4SE3Nzer6aVKlbLMj/+vvb29ihUrZtWuZMmSVq9v376te/fuafny5Vq+fHmitd2+fTvF+wUAQEZBwAYA4Dni5+enDz74QDdv3lT9+vWVI0eONN9mXFycJKlt27bq0KFDom28vLzSvA4AAGyNgA0AwHOkadOm+vDDD3X48GFNmzYt0TaFCxfW3r17FRoaanUW+8yZM5b58f+Ni4vThQsXrM5ax7eL5+7urmzZsikuLk516tQxe5cAAMgwuAYbAIDnSLZs2TRu3DgNGTJEjRs3TrRN/fr1FRsbqyVLllhN//bbb2VnZ6f69etb2kmyugO5JP3vf/+zeu3g4KDmzZtr06ZN+vvvvxNsj+HhAIDMgjPYAAA8Zx43TDte48aNVbNmTU2bNk2XL1+Wl5eXdu/erW3btqlnz56Wa67LlSsnPz8/ff/99woJCVGVKlW0b98+nT9/PsE633nnHe3fv1+dO3dWp06dVLp0aQUHB+v48ePau3evDhw4kCb7CgBAekLABgAgk7G3t9fcuXM1c+ZMrV+/XqtXr1bhwoU1YsQI9e7d26rtxIkTlTt3bgUEBGjbtm2qWbOmvvrqK8uztuPlzZtXP/zwg+bMmaMtW7Zo6dKlypUrl0qXLq3hw4c/y90DAMBm7AzDMGxdBAAAAAAAGR3XYAMAAAAAYAICNgAAAAAAJiBgAwAAAABgAgI2AAAAAAAmIGADAAAAAGACAjYAAAAAACYgYAMAAAAAYAICNgAAAAAAJiBgAwAAAABgAgI2AAAAAAAmIGADAAAAAGACAjYAAAAAACYgYAMAAAAAYIL/B0KDfLTtrnNKAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Data for plotting\n",
    "methods = ['Baseline (ins-last)', 'ins-first', 't2t', 'modeling prompt']\n",
    "A = [70.26, 73.22, 69.13, 63.21]\n",
    "B = [72.00, 73.50, 60.00, 66.00]\n",
    "C = [74.87, 76.08, 71.92, 61.87]\n",
    "\n",
    "# Data preparation\n",
    "data = {\n",
    "    'Method': methods * 3,\n",
    "    'Percentage': A + B + C,\n",
    "    'Model': ['InternLM-Chat'] * len(A) + ['Baichuan2-Chat'] * len(B) + ['LLaMA-Alpaca'] * len(C)\n",
    "}\n",
    "s\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Setting seaborn style\n",
    "sns.set(style=\"whitegrid\")\n",
    "\n",
    "# Create the plot\n",
    "plt.figure(figsize=(10, 6))\n",
    "ax = sns.barplot(x='Model', y='Percentage', hue='Method', data=df)\n",
    "\n",
    "# Customizing the plot\n",
    "# ax.set_xlabel('Group', fontsize=14)\n",
    "ax.set_ylabel('Percentage (%)', fontsize=14)\n",
    "# ax.set_title('Comparison of Different Methods across Groups A, B, and C', fontsize=16)\n",
    "ax.set_ylim(50, 82)  # Setting y-axis limits\n",
    "plt.legend(title='Method', loc='upper center', bbox_to_anchor=(0.5, 1), ncol=len(methods))  # Centered below the plot\n",
    "plt.setp(ax.get_xticklabels(), fontsize=12)\n",
    "plt.setp(ax.get_yticklabels(), fontsize=12)\n",
    "sns.despine(bottom=True, left=True)\n",
    "\n",
    "# Adjust layout\n",
    "plt.tight_layout(rect=[0, 0, 1, 1])\n",
    "\n",
    "# Show the plot\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gby",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}