From eeb031ea9049498f4781ca001c1b9085ca93b7a7 Mon Sep 17 00:00:00 2001 From: Ivan Charapanau Date: Sun, 6 Oct 2024 16:29:09 +0200 Subject: [PATCH] feat: boost - whole bunch of experimental custom modules, slight improvements to chat/llm APIs, better error for missing Auth, modules can now accept params via completion request body with "@boost" prefixed keys --- boost/src/chat.py | 16 +++ boost/src/chat_node.py | 19 +-- boost/src/custom_modules/3t.py | 43 ++++++ boost/src/custom_modules/ambi.py | 100 ++++++++++++++ boost/src/custom_modules/cea.py | 94 +++++++++++++ boost/src/custom_modules/clarity.py | 58 ++++++++ boost/src/custom_modules/fml.py | 33 +++++ boost/src/custom_modules/l33t.py | 2 +- boost/src/custom_modules/pad.py | 207 ++++++++++++++++++++++++++++ boost/src/llm.py | 13 +- boost/src/main.py | 14 +- boost/src/selection.py | 9 +- 12 files changed, 586 insertions(+), 22 deletions(-) create mode 100644 boost/src/custom_modules/3t.py create mode 100644 boost/src/custom_modules/ambi.py create mode 100644 boost/src/custom_modules/cea.py create mode 100644 boost/src/custom_modules/clarity.py create mode 100644 boost/src/custom_modules/fml.py create mode 100644 boost/src/custom_modules/pad.py diff --git a/boost/src/chat.py b/boost/src/chat.py index 41d1521..a988faf 100644 --- a/boost/src/chat.py +++ b/boost/src/chat.py @@ -15,6 +15,10 @@ def from_conversation(messages): tail = ChatNode.from_conversation(messages) return Chat(tail=tail) + def from_tail(chat): + new_tail = ChatNode(role=chat.tail.role, content=chat.tail.content) + return Chat(tail=new_tail) + def __init__(self, **kwargs): self.tail = kwargs.get('tail') self.llm = kwargs.get('llm') @@ -83,5 +87,17 @@ async def emit_advance(self): response = await self.llm.stream_chat_completion(chat=self) self.assistant(response) + async def emit_status(self, status): + """ + Emit a status message + + Will be streamed back to the client + """ + + if not self.llm: + raise ValueError("Chat: unable to emit status without an LLM") + + await self.llm.emit_status(status) + def __str__(self): return '\n'.join([str(msg) for msg in self.plain()]) diff --git a/boost/src/chat_node.py b/boost/src/chat_node.py index e513e3d..8020ff9 100644 --- a/boost/src/chat_node.py +++ b/boost/src/chat_node.py @@ -5,6 +5,7 @@ logger = log.setup_logger(__name__) + class ChatNode: id: str content: str @@ -70,6 +71,12 @@ def parents(self): return parents[::-1] + def message(self): + return { + "role": self.role, + "content": self.content, + } + def ancestor(self): node = self while node.parent: @@ -78,19 +85,13 @@ def ancestor(self): def history(self): node = self - messages = [{ - "role": node.role, - "content": node.content, - }] + messages = [node.message()] while node.parent: node = node.parent - messages.append({ - "role": node.role, - "content": node.content, - }) + messages.append(node.message()) return messages[::-1] def __str__(self): - return f"{self.role}: {self.content}" \ No newline at end of file + return f"{self.role}: {self.content}" diff --git a/boost/src/custom_modules/3t.py b/boost/src/custom_modules/3t.py new file mode 100644 index 0000000..0d141f6 --- /dev/null +++ b/boost/src/custom_modules/3t.py @@ -0,0 +1,43 @@ +import chat as ch +import llm + +ID_PREFIX = '3t' + + +async def apply(chat: 'ch.Chat', llm: 'llm.LLM'): + side_chat = ch.Chat( + tail=ch.ChatNode( + content=""" +I will ask you to answer my question three times. Each time you will provide a different answer. +Try to use the chance to correct any mistakes you made in the previous answers. + """.strip() + ) + ) + side_chat.llm = llm + + side_chat.user('Here is the question:') + side_chat.user(chat.tail.content) + side_chat.user('Please provide the first answer to the question.') + await side_chat.emit_status('First') + await side_chat.emit_advance() + + side_chat.user( + 'Please provide the second answer to the question. Remember, it must be different from the first one.' + ) + await side_chat.emit_status('Second') + await side_chat.emit_advance() + + side_chat.user( + 'Please provide the third answer to the question. It must be different from the first two.' + ) + await side_chat.emit_status('Third') + await side_chat.emit_advance() + + side_chat.user( + """ +Now, think about the answers you provided. Is there anything wrong with them? Which one is the most correct? +What is the final answer to the question? + """.strip() + ) + await side_chat.emit_status('Final') + await llm.stream_final_completion(chat=side_chat) diff --git a/boost/src/custom_modules/ambi.py b/boost/src/custom_modules/ambi.py new file mode 100644 index 0000000..0fea4cf --- /dev/null +++ b/boost/src/custom_modules/ambi.py @@ -0,0 +1,100 @@ +import chat as ch +import llm + +ID_PREFIX = 'ambi' + +ambi_prompt = """ + +Find the sources of ambiguities in the given question and describe them. + + + +{question} + + """.strip() + +detail_prompt = """ + +Find the conditions that significantly affect the interpretation of the question and describe them. + + + +{question} + +""".strip() + +definition_prompt = """ + +Define the terms in the question and provide a detailed explanation for each. + + + +{question} + +""".strip() + +discrepancies_prompt = """ + +Find the discrepancies in the question and describe them. + + + +{question} + +""".strip() + +final_prompt = """ + +Provide a clear and definitive answer to the question. + + + +{question} + + + +### Ambiguities +{ambiguities} + +### Details +{details} + +### Definitions +{definitions} + +### Discrepancies +{discrepancies} + +""".strip() + + +async def apply(chat: 'ch.Chat', llm: 'llm.LLM'): + await llm.emit_status('Ambiguiity') + ambiguities = await llm.stream_chat_completion( + prompt=ambi_prompt, question=chat.tail.content + ) + + await llm.emit_status('Details') + details = await llm.stream_chat_completion( + prompt=detail_prompt, question=chat.tail.content + ) + + await llm.emit_status('Definitions') + definitions = await llm.stream_chat_completion( + prompt=definition_prompt, question=chat.tail.content + ) + + await llm.emit_status('Discrepancies') + discrepancies = await llm.stream_chat_completion( + prompt=discrepancies_prompt, question=chat.tail.content + ) + + await llm.emit_status('Final') + await llm.stream_final_completion( + prompt=final_prompt, + question=chat.tail.content, + ambiguities=ambiguities, + details=details, + definitions=definitions, + discrepancies=discrepancies + ) diff --git a/boost/src/custom_modules/cea.py b/boost/src/custom_modules/cea.py new file mode 100644 index 0000000..eef4d74 --- /dev/null +++ b/boost/src/custom_modules/cea.py @@ -0,0 +1,94 @@ +import random + +import chat as ch +import llm +import log + +ID_PREFIX = 'cea' # 'cellular automata' + + +def cellular_automata(rule, initial_state, generations): + """ + Runs a one-dimensional cellular automata and records results in binary, + allowing the state to grow. + + Args: + rule (int): The rule number for the cellular automata (0-255). + initial_state (list): The initial state of the cellular automata. + generations (int): The number of generations to run. + + Returns: + list: A list of binary strings representing the state of the cellular automata at each generation. + """ + # Convert the rule number to a binary string and pad with zeros to 8 bits + rule_binary = format(rule, '08b') + + # Initialize the list to store the results + results = ["".join(map(str, initial_state))] + + # Run the cellular automata for the specified number of generations + current_state = initial_state.copy() + for _ in range(generations): + # Initialize the next state with a zero on each end + next_state = [0] + current_state + [0] + + # Apply the rule to each cell in the current state + for i in range(1, len(next_state) - 1): + # Get the left, center, and right cells + left = current_state[i - 2] if i > 1 else 0 + center = current_state[i - 1] + right = current_state[i] if i < len(current_state) else 0 + + # Convert the left, center, and right cells to a binary string + neighborhood = f"{left}{center}{right}" + + # Get the next state of the cell based on the rule + next_state[i] = int(rule_binary[7 - int(neighborhood, 2)]) + + # Update the current state and append the next state to the results + current_state = next_state + results.append("".join(map(str, next_state))) + + return results + +def render_ca(results): + """ + Renders the results of a cellular automata as a string. + + Args: + results (list): A list of binary strings representing the state of the cellular automata at each generation. + + Returns: + str: A string representation of the cellular automata results. + """ + return join.join(["".join(["|" if cell == "1" else "." for cell in result]) for result in results]) + + + +initial_state = [1] +join = '\n' + + +async def apply(chat: 'ch.Chat', llm: 'llm.LLM'): + rule = int(llm.boost_params.get('cea_rule', '73')) + gens = int(llm.boost_params.get('cea_generations', '32')) + + chat.user( + f""" +Before completing my request, please think for a while. + """.strip() + ) + chat.assistant( + f"""Good idea! Let me think... + +```thoughts +{render_ca(cellular_automata(rule, initial_state, gens))} +``` + +""" + ) + await llm.emit_message(chat.tail.content) + chat.user(f""" +Now, please address my request. + """.strip()) + await llm.stream_final_completion() diff --git a/boost/src/custom_modules/clarity.py b/boost/src/custom_modules/clarity.py new file mode 100644 index 0000000..5ef1e15 --- /dev/null +++ b/boost/src/custom_modules/clarity.py @@ -0,0 +1,58 @@ +import chat as ch +import llm + +ID_PREFIX = 'clarity' + +should_clarify_prompt = """ + +Is this question requires any clarification or is ready to be answered? +Reply only with "clarify" or "ready" and nothing else. Everything else will be ignored. + + + +{question} + + """.strip() + + +async def apply(chat: 'ch.Chat', llm: 'llm.LLM'): + iterations = 0 + max_iterations = 15 + + side_chat = ch.Chat.from_conversation([chat.tail.message()]) + side_chat.llm = llm + + while iterations < max_iterations: + iterations += 1 + side_chat.user( + """ +Are there any sources of ambiguity in my request? +Answer with "yes" or "no" and nothing else. Everything else will be ignored. + """.strip() + ) + await side_chat.advance() + await llm.emit_status(f'Clarification: {side_chat.tail.content}') + + if side_chat.tail.contains('no'): + break + + side_chat.user(""" +Clarify the ambiguity you mentioned. + """.strip()) + await side_chat.emit_advance() + + if iterations >= max_iterations: + break + + side_chat.user('Now, please provide a clear answer to the question.') + await side_chat.emit_advance() + + await llm.emit_status('Final') + + side_chat.user( + """ +Think trough the response you just gave. Is there anything wrong? If so, please correct it. +Otherwise, write down your final answer to my request. + """.strip() + ) + await llm.stream_final_completion(chat=chat) \ No newline at end of file diff --git a/boost/src/custom_modules/fml.py b/boost/src/custom_modules/fml.py new file mode 100644 index 0000000..62b2b85 --- /dev/null +++ b/boost/src/custom_modules/fml.py @@ -0,0 +1,33 @@ +import random + +import chat as ch +import llm +import log + +ID_PREFIX = 'fml' # "formulaic", not what you think + +async def apply(chat: 'ch.Chat', llm: 'llm.LLM'): + chat.user( + f""" +Rewrite my request in the formulaic logic language. Do not solve it yet. + """.strip() + ) + await chat.emit_status('Formulaic') + await chat.emit_advance() + + chat.user( + f""" +Solve my original request in the formulaic logic language. +""".strip() + ) + await chat.emit_status('Solution') + await chat.emit_advance() + + chat.user( + f""" +Rewrite it in the natural language. +""".strip() + ) + + await chat.emit_status('Final') + await llm.stream_final_completion() diff --git a/boost/src/custom_modules/l33t.py b/boost/src/custom_modules/l33t.py index e820dc7..47e5627 100644 --- a/boost/src/custom_modules/l33t.py +++ b/boost/src/custom_modules/l33t.py @@ -4,7 +4,7 @@ ID_PREFIX = 'l33t' async def apply(chat: 'ch.Chat', llm: 'llm.LLM'): - side_chat = ch.Chat.from_conversation([chat.history()[-1]]) + side_chat = ch.Chat.from_conversation([chat.tail.message()]) side_chat.llm = llm await llm.emit_status('l33t speak...') diff --git a/boost/src/custom_modules/pad.py b/boost/src/custom_modules/pad.py new file mode 100644 index 0000000..589385e --- /dev/null +++ b/boost/src/custom_modules/pad.py @@ -0,0 +1,207 @@ +import random + +import chat as ch +import llm +import log + +ID_PREFIX = 'pad' + +THINKING = [ + 'Actively thinking...', + 'Considering possible implications...', + 'Simplifying...', + 'Ensuring correctness...', + 'Clarifying...', + 'Rephrasing a thought...', + 'Reconsidering...', + 'Evaluating assumption...', + 'Analyzing constraints...', + 'Reflecting on a solution...', + 'Reviewing...', + 'Contemplating...', + 'Pondering...', + 'Speculating...', + 'Deliberating...', + 'Focusing on the outcome...', + 'Imagining alternatives...', + 'Envisioning a simpler path...', + 'Creating a more concise outline...', + 'Constructing a more elaborate plan...', + 'Designing a more efficient solution...', + 'Inventing a more effective strategy...', + 'Devising a more practical approach...', + 'Formulating a more sophisticated method...', + 'Developing a more advanced technique...', +] + +THINKING_2 = [ + 'Thinking about task at hand', + 'Applying critical thinking', + 'Choosing more practical options', + 'Ensuring pragmatic solutions', + 'Simplifying', + 'Ensuring correctness', + 'Ensuring practicality' + 'Removing sources of ambiguity', + 'Making sure it makes sense', + 'Ensuring clarity', + 'Ensuring simplicity', + 'Making sure it\'s easy to understand', + 'Verifying the logic', + 'Checking for errors', + 'Making sure I did not miss anything', + 'Avoiding obvious mistakes', + 'Fixing an error', + 'Correcting a mistake', + 'Breaking down the problem', + 'Ensuring the solution is feasible', + 'Clarifying assumptions', +] + +WORDS = [ + 'apple', 'banana', 'cherry', 'date', 'elderberry', 'fig', 'grape', 'honeydew', + 'kiwi', 'lemon', 'mango', 'nectarine', 'orange', 'pear', 'quince', + 'raspberry', 'strawberry', 'tangerine', 'ugli', 'vanilla', 'watermelon', + 'ximenia', 'yuzu', 'zucchini' +] + + +def get_size(**kwargs): + return int(kwargs.get('pad_size', 256)) + + +def pad_thinking(**kwargs): + size = get_size(**kwargs) + join = kwargs.get('join', '\n') + return join.join([random.choice(THINKING_2) for _ in range(size)]) + + +def pad_newline(**kwargs): + size = get_size(**kwargs) + return '\n' * size + + +def pad_space(**kwargs): + size = get_size(**kwargs) + return ' ' * size + + +def pad_random_nl(**kwargs): + size = get_size(**kwargs) + return ''.join([random.choice([' ', '\n']) for _ in range(size)]) + + +def random_alphabet(): + return random.choice('abcdefghijklmnopqrstuvwxyz') + + +def pad_random_alphabet(**kwargs): + size = get_size(**kwargs) + join = kwargs.get('join', '') + pad = join.join([random_alphabet() for _ in range(size)]) + return pad + + +def pad_random_words(**kwargs): + size = get_size(**kwargs) + pad = ' '.join([random.choice(WORDS) for _ in range(size)]) + return pad + + +def pad_random_numbers(**kwargs): + size = get_size(**kwargs) + pad = ' '.join([str(random.randint(0, 9)) for _ in range(size)]) + return pad + + +def pad_thinking_loop(**kwargs): + size = get_size(**kwargs) + join = kwargs.get('join', '\n') + return join.join([THINKING_2[i % len(THINKING_2)] for i in range(size)]) + +def pad_thinking_steps(**kwargs): + size = get_size(**kwargs) + join = kwargs.get('join', '\n') + + return join.join([f'Step {i}: {THINKING_2[i % len(THINKING_2)]}...' for i in range(size)]) + +def pad_random_thinking_steps(**kwargs): + size = get_size(**kwargs) + join = kwargs.get('join', '\n') + + return join.join([f'Step {i}: {random.choice(THINKING_2)}...' for i in range(size)]) + + +PAD_TYPES = { + 'thinking': pad_thinking, + 'newline': pad_newline, + 'space': pad_space, + 'random_nl': pad_random_nl, + 'random_alphabet': pad_random_alphabet, + 'random_words': pad_random_words, + 'random_numbers': pad_random_numbers, + 'thinking_loop': pad_thinking_loop, + 'thinking_steps': pad_thinking_steps, + 'random_thinking_steps': pad_random_thinking_steps, +} + +PAD_STYLES = { + 'plain': lambda x: x, + 'block': lambda x: f""" +```entropy +{x} +``` +""".strip(), + 'block_thoughts': lambda x: f""" +```thoughts +Starting internal thought process... +{x} +Ok, I am ready for the final answer now. +``` +""".strip(), + 'quote': lambda x: f""" +> {x} +""".strip(), +} + + +def make_pad(**kwargs): + pad_type = kwargs.get('pad_type', 'random_thinking_steps') + pad_style = kwargs.get('pad_style', 'block_thoughts') + + return PAD_STYLES[pad_style](PAD_TYPES[pad_type](**kwargs)) + + +logger = log.setup_logger(__name__) + + +async def apply(chat: 'ch.Chat', llm: 'llm.LLM'): + llm.boost_params['pad_size'] = '128' + + pad = make_pad(**llm.boost_params) + + chat.user( + f""" +Before addresing my request, I need you to take your time and think for a while. +It's very important for you to utilise this time to concentrate on the task at hand. +""" + ) + + await chat.emit_status('Thinking...') + + chat.assistant( + f""" +Thank you for letting me think for a bit! I will use this time to concentrate on the task at hand. +{pad} + """ + ) + await llm.emit_message(chat.tail.content) + + chat.user( + f""" +Ok, I think we're ready now. Please answer my previous request. + """ + ) + + await chat.emit_status('Final') + await llm.stream_final_completion() diff --git a/boost/src/llm.py b/boost/src/llm.py index fd3bfc6..0d279eb 100644 --- a/boost/src/llm.py +++ b/boost/src/llm.py @@ -14,6 +14,8 @@ logger = log.setup_logger(__name__) +BOOST_PARAM_PREFIX="@boost_" + class LLM: url: str @@ -21,6 +23,7 @@ class LLM: model: str params: dict + boost_params: dict module: str queue: asyncio.Queue @@ -34,7 +37,7 @@ def __init__(self, **kwargs): self.headers = kwargs.get('headers', {}) self.model = kwargs.get('model') - self.params = kwargs.get('params', {}) + self.split_params(kwargs.get('params', {})) self.chat = self.resolve_chat(**kwargs) self.messages = self.chat.history() @@ -51,6 +54,14 @@ def __init__(self, **kwargs): def chat_completion_endpoint(self): return f"{self.url}/chat/completions" + def split_params(self, params: dict): + self.params = { + k: v for k, v in params.items() if not k.startswith(BOOST_PARAM_PREFIX) + } + self.boost_params = { + k[len(BOOST_PARAM_PREFIX):]: v for k, v in params.items() if k.startswith(BOOST_PARAM_PREFIX) + } + def generate_system_fingerprint(self): return "fp_boost" diff --git a/boost/src/main.py b/boost/src/main.py index 2a7bd8b..5d630ba 100644 --- a/boost/src/main.py +++ b/boost/src/main.py @@ -22,10 +22,13 @@ async def get_api_key(api_key_header: str = Security(auth_header)): if len(BOOST_AUTH) == 0: return - # Bearer/plain versions - value = api_key_header.replace("Bearer ", "").replace("bearer ", "") - if value in BOOST_AUTH: - return value + + if api_key_header is not None: + # Bearer/plain versions + value = api_key_header.replace("Bearer ", "").replace("bearer ", "") + if value in BOOST_AUTH: + return value + raise HTTPException(status_code=403, detail="Unauthorized") @@ -67,6 +70,7 @@ async def get_boost_models(api_key: str = Depends(get_api_key)): if should_filter: should_serve = selection.matches_filter(model, MODEL_FILTER.value) + print(model['id'], MODEL_FILTER.value['id.regex'], should_serve) if should_serve: final.append(model) @@ -122,7 +126,7 @@ async def post_boost_chat_completion(request: Request, api_key: str = Depends(ge logger.info(f"Boosting: {config.BOOST_APIS}") if len(BOOST_AUTH) == 0: - logger.warn("No API keys specified - boost will accept all requests") + logger.warn("No API keys specified - boost will accept all requests") if __name__ == "__main__": import uvicorn diff --git a/boost/src/selection.py b/boost/src/selection.py index 399214c..8162e21 100644 --- a/boost/src/selection.py +++ b/boost/src/selection.py @@ -82,13 +82,10 @@ def matches_filter(obj: dict, filter: dict): return False if operation == 'regex': - if not match_regex(str(obj[field]), value): - return False + return match_regex(str(obj[field]), value) elif operation == 'contains': - if not match_substring(str(obj[field]), value): - return False + return match_substring(str(obj[field]), value) else: - if not match_exact(str(obj[field]), value): - return False + return match_exact(str(obj[field]), value) return True \ No newline at end of file