chatbot.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Kyouma960/ChatBot/blob/main/llama.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZBzEYWn189Nc"
      },
      "outputs": [],
      "source": [
        "import locale\n",
        "def getpreferredencoding(do_setlocale = True):\n",
        "    return \"UTF-8\"\n",
        "locale.getpreferredencoding = getpreferredencoding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bHEqkVxaRYKa"
      },
      "outputs": [],
      "source": [
        "!pip install nest_asyncio\n",
        "!pip install discord.py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1kIjCrN2Vqgb"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/tloen/alpaca-lora && cd alpaca-lora && pip install -r requirements.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fHRPqxcbXCoi"
      },
      "outputs": [],
      "source": [
        "!pip install torch==2.0.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0Z96uHZlJrd9"
      },
      "outputs": [],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iYYNPcY98cjI"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from peft import PeftModel\n",
        "import transformers\n",
        "import textwrap\n",
        "from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig\n",
        "from transformers.generation.utils import GreedySearchDecoderOnlyOutput"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kJ_ThPS2FfR5"
      },
      "outputs": [],
      "source": [
        "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "DEVICE"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jysAghwyGMxQ"
      },
      "outputs": [],
      "source": [
        "tokenizer = LlamaTokenizer.from_pretrained(\"decapoda-research/llama-7b-hf\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v8T-FTEXHD-c"
      },
      "outputs": [],
      "source": [
        "model = LlamaForCausalLM.from_pretrained(\n",
        "  \"decapoda-research/llama-7b-hf\",\n",
        "  load_in_8bit=True,\n",
        "  device_map=\"auto\"\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9U-Fkf9bH4P8"
      },
      "outputs": [],
      "source": [
        "model = PeftModel.from_pretrained(\n",
        "    model,\n",
        "    \"tloen/alpaca-lora-7b\",\n",
        "    torch_dtype=torch.float16\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pdZwn0MlIq9G"
      },
      "outputs": [],
      "source": [
        "model.config.pad_token_id = tokenizer.pad_token_id =0\n",
        "model.config.bos_token_id = 1\n",
        "model.config.eos_token_id = 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wSrtyPr0JCCZ"
      },
      "outputs": [],
      "source": [
        "model = model.eval()\n",
        "model = torch.compile(model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HcgCcCV4JRW-"
      },
      "outputs": [],
      "source": [
        "PROMPT_TEMPLATE = f\"\"\"\n",
        "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
        "\n",
        "### Instruction:\n",
        "[INSTRUCTION]\n",
        "\n",
        "### Response:\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hTHb7_xqWn4w"
      },
      "outputs": [],
      "source": [
        "PROMPT_INPUT_TEMPLATE = f\"\"\"\n",
        "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n",
        "\n",
        "### Instruction:\n",
        "[INSTRUCTION]\n",
        "\n",
        "### Input:\n",
        "[INPUT]\n",
        "\n",
        "### Response:\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H6nC7RKZJc6Z"
      },
      "outputs": [],
      "source": [
        "def create_prompt(instruction: str) -> str:\n",
        "  return PROMPT_TEMPLATE.replace(\"[INSTRUCTION]\", instruction)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E5RpmfvZQ14t"
      },
      "outputs": [],
      "source": [
        "def generate_response(prompt: str, model: PeftModel) -> GreedySearchDecoderOnlyOutput:\n",
        "  encoding = tokenizer(prompt, return_tensors=\"pt\")\n",
        "  input_ids = encoding[\"input_ids\"].to(DEVICE)\n",
        "  generation_config = GenerationConfig(\n",
        "    temperature = 0.1,\n",
        "    top_p=0.75,\n",
        "    repetition_penalty=1.1\n",
        "  )\n",
        "  with torch.inference_mode():\n",
        "      global output\n",
        "      output = model.generate(\n",
        "          input_ids=input_ids,\n",
        "          generation_config=generation_config,\n",
        "          return_dict_in_generate=True,\n",
        "          output_scores=True,\n",
        "          max_new_tokens=1024\n",
        "      )\n",
        "      return output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jS8k_1OnR8VY"
      },
      "outputs": [],
      "source": [
        "def format_response(response: GreedySearchDecoderOnlyOutput) -> str:\n",
        "  decoded_output = tokenizer.decode(output.sequences[0])\n",
        "  response = decoded_output.split(\"### Response:\")[1].strip()\n",
        "  return \"\\n\".join(textwrap.wrap(response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fVpwrjkiTDFH"
      },
      "outputs": [],
      "source": [
        "def ask_alpaca(prompt: str, model: PeftModel = model) -> str:\n",
        "  prompt = create_prompt(prompt)\n",
        "  response = generate_response(prompt, model)\n",
        "  return (format_response(response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XtiuxeWUfG87"
      },
      "outputs": [],
      "source": [
        "#@markdown #**Anti-Disconnect for Google Colab**\n",
        "#@markdown ## Run this to stop it from disconnecting automatically\n",
        "#@markdown  **(It will anyhow disconnect after 6 - 12 hrs for using the free version of Colab.)**\n",
        "#@markdown  *(Colab Pro users will get about 24 hrs usage time)*\n",
        "#@markdown ---\n",
        "\n",
        "import IPython\n",
        "js_code = '''\n",
        "function ClickConnect(){\n",
        "console.log(\"Working\");\n",
        "document.querySelector(\"colab-toolbar-button#connect\").click()\n",
        "}\n",
        "setInterval(ClickConnect,60000)\n",
        "'''\n",
        "display(IPython.display.Javascript(js_code))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wKIHeVxzo5rv"
      },
      "outputs": [],
      "source": [
        "import discord\n",
        "from discord.ext import commands\n",
        "import nest_asyncio\n",
        "nest_asyncio.apply()\n",
        "TOKEN = ''\n",
        "intents = discord.Intents.default()\n",
        "intents.message_content = True\n",
        "\n",
        "client = commands.Bot(command_prefix='reg ', intents=intents)\n",
        "@client.event\n",
        "async def on_ready():\n",
        "    print('bot ready')\n",
        "\n",
        "@client.event\n",
        "async def on_message(message):\n",
        "    if message.author == client.user:\n",
        "        return\n",
        "\n",
        "    if message.content.startswith('reg ask '):\n",
        "        user_message = message.content[8:]\n",
        "        answer = ask_alpaca(user_message)\n",
        "        embedVar = discord.Embed(title=user_message, description=answer, color=0x00ff00)\n",
        "        await message.channel.send(embed=embedVar)\n",
        "        #await message.channel.send(answer)\n",
        "\n",
        "client.run(TOKEN)\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}