From 5677f674c1a22400e8b28c0d954295866754332b Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Thu, 7 Mar 2024 08:59:45 +0000 Subject: [PATCH 1/4] add answers for invalid and not enough info to resolve market --- .../resolve_market_reasoning.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py b/packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py index fb5ea427..c2e90f11 100644 --- a/packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py +++ b/packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py @@ -158,6 +158,12 @@ class Date(OpenAISchema): class Results(OpenAISchema): has_occurred: bool = Field(..., description="Whether the event has occurred.") +class Valid(OpenAISchema): + is_valid: bool = Field(..., description="Whether the question is valid.") + reason: Optional[str] = Field(..., description="Reason that the question is invalid.") + +class Determinable(OpenAISchema): + is_determinable: bool = Field(..., description="Whether it is possible to answer the question based on the information provided and reasoning.") class Document(BaseModel): text: str @@ -261,6 +267,45 @@ class Document(BaseModel): ``` """ +VALID_PROMPT = """ +* You are an expert data analyst. +* You are provided with a question about an event (submitted to a prediction market) under "USER_PROMPT" delimited by three backticks. +* Your task is to determine whether the question is valid. +* You are provided with rules that determine whether a question is invalid (as well as examples) under the label "RULES". +* Your response should only be a function call with the information about whether a question is valid and reason as arguments. + +RULES +* Questions with relative dates should be marked as invalid. E.g. Invalid: Who will be the president of the United States in 6 months? (“in 6 months depends on the current time”). +* Questions about moral values and not facts should be marked as invalid. E.g. Invalid: “Is it ethical to eat meat?”. +* Questions in which none of the answers are valid should be marked as invalid. E.g. Invalid: “What is the result of 1+1?” with the outcomes “0” and “1”. +* Questions in which multiple answers are valid should be marked as invalid. E.g. Invalid: “Who will be the Time person of the year 1937?” with answers “Chiang Kai-shek” and “Soong Mei-ling” (they got the prize jointly). + +USER_PROMPT: +``` +{user_prompt} +``` +""" + +DETERMINABLE_PROMPT = """ +* You are an expert data analyst. +* You are provided with a question about an event (submitted to a prediction market) under "USER_PROMPT" delimited by three backticks. +* You are provided with a colleague's reasoning as to whether the event occurred based on online research under the label "REASONING" delimited by three backticks. +* Your task is to determine whether it is possible to answer whether the event actually happened before the date based on the content of this reasoning. +* The answer that you give should reflect the opinion in the reasoning field. +* Your response should only be a function call with the information about whether a question is valid and reason as arguments. + +USER_PROMPT: +``` +{user_prompt} +``` + +REASONING: +``` +{reasoning} +``` + +""" + SYSTEM_PROMPT = """You are a world class algorithm for generating structured output from a given input.""" @@ -628,6 +673,34 @@ def run(**kwargs) -> Tuple[Optional[str], Optional[Dict[str, Any]], Any]: engine = TOOL_TO_ENGINE[tool] + # Check if question is valid + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + { + "role": "user", + "content": VALID_PROMPT.format( + user_prompt=prompt + ), + }, + ] + + response_valid = client.chat.completions.create( + model=engine, + messages=messages, + temperature=DEFAULT_OPENAI_SETTINGS["temperature"], + max_tokens=DEFAULT_OPENAI_SETTINGS["max_tokens"], + n=1, + timeout=150, + stop=None, + functions=[Valid.openai_schema], + ) + + valid_results = Valid.from_response(response_valid) + print(f"Valid: {valid_results}") + + if not valid_results.is_valid: + return valid_results, None, None, None, None + try: ( additional_information, @@ -672,6 +745,34 @@ def run(**kwargs) -> Tuple[Optional[str], Optional[Dict[str, Any]], Any]: reasoning = response_reasoning.choices[0].message.content + # Check if question is determinable + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + { + "role": "user", + "content": DETERMINABLE_PROMPT.format( + user_prompt=prompt, reasoning=reasoning + ), + }, + ] + + response_determinable = client.chat.completions.create( + model=engine, + messages=messages, + temperature=DEFAULT_OPENAI_SETTINGS["temperature"], + max_tokens=DEFAULT_OPENAI_SETTINGS["max_tokens"], + n=1, + timeout=150, + stop=None, + functions=[Determinable.openai_schema], + ) + + determinable_results = Determinable.from_response(response_determinable) + print(f"Determinable: {determinable_results}") + + if not determinable_results.is_determinable: + return determinable_results, reasoning, additional_information, queries, None + # Make the prediction messages = [ {"role": "system", "content": SYSTEM_PROMPT}, From 5b48ec9da3ee8a04ef8f451ded7e5ec2bcd938b9 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Thu, 7 Mar 2024 16:10:11 +0000 Subject: [PATCH 2/4] fix hashes --- .../napthaai/customs/resolve_market_reasoning/component.yaml | 2 +- packages/packages.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/napthaai/customs/resolve_market_reasoning/component.yaml b/packages/napthaai/customs/resolve_market_reasoning/component.yaml index 23828892..716fb2ff 100644 --- a/packages/napthaai/customs/resolve_market_reasoning/component.yaml +++ b/packages/napthaai/customs/resolve_market_reasoning/component.yaml @@ -7,7 +7,7 @@ license: Apache-2.0 aea_version: '>=1.0.0, <2.0.0' fingerprint: __init__.py: bafybeib36ew6vbztldut5xayk5553rylrq7yv4cpqyhwc5ktvd4cx67vwu - resolve_market_reasoning.py: bafybeickfol4s4kilpxoanphodizenft4ldhuuuieewyvfx5gymig3x644 + resolve_market_reasoning.py: bafybeia55z2cdeqfa2ca7p6kptz6qsyjoj5x4vmrtyt4ju7oau333i6cya fingerprint_ignore_patterns: [] entry_point: resolve_market_reasoning.py callable: run diff --git a/packages/packages.json b/packages/packages.json index 1a30e2f5..483b0987 100644 --- a/packages/packages.json +++ b/packages/packages.json @@ -12,7 +12,7 @@ "custom/psouranis/optimization_by_prompting/0.1.0": "bafybeibgfovquvu3gg4o7y6ud2sr536v45o2dj4shqobc7bqzzdyjbinei", "custom/nickcom007/sme_generation_request/0.1.0": "bafybeihfl4663yjkxltidavnxou7rumld5wzcr43uw23bpvb3ivefsjhv4", "custom/nickcom007/prediction_request_sme/0.1.0": "bafybeicfenfjer2kvmnstul4bzxf7xfiupbipkjzattjsieaapdflnsyni", - "custom/napthaai/resolve_market_reasoning/0.1.0": "bafybeifkg7l42txwa2cwvzlhxjvrtiupskh2h3ojlh4i275wksxoqxhfp4", + "custom/napthaai/resolve_market_reasoning/0.1.0": "bafybeialxdi5btf2kx4auce5sdpjzd2rea65v2l4ivmr4j5ignpum7vs54", "custom/napthaai/prediction_request_rag/0.1.0": "bafybeihdywbiaywvia32mpdos7e4pvcnmsq4ij65otctidbd6rjf3j6tqu", "protocol/valory/acn_data_share/0.1.0": "bafybeih5ydonnvrwvy2ygfqgfabkr47s4yw3uqxztmwyfprulwfsoe7ipq", "protocol/valory/websocket_client/0.1.0": "bafybeih43mnztdv3v2hetr2k3gezg7d3yj4ur7cxdvcyaqhg65e52s5sf4", From d6ed478d93153d3acab49b564bb08b8afa29f836 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Fri, 8 Mar 2024 14:29:49 +0000 Subject: [PATCH 3/4] return results as json string --- .../resolve_market_reasoning/resolve_market_reasoning.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py b/packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py index c2e90f11..85e402af 100644 --- a/packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py +++ b/packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py @@ -27,6 +27,7 @@ from pydantic import BaseModel, Field from docstring_parser import parse import tiktoken +import json from openai import OpenAI import numpy as np import faiss @@ -699,7 +700,7 @@ def run(**kwargs) -> Tuple[Optional[str], Optional[Dict[str, Any]], Any]: print(f"Valid: {valid_results}") if not valid_results.is_valid: - return valid_results, None, None, None, None + return valid_results.json(), None, None, None, None try: ( @@ -771,7 +772,7 @@ def run(**kwargs) -> Tuple[Optional[str], Optional[Dict[str, Any]], Any]: print(f"Determinable: {determinable_results}") if not determinable_results.is_determinable: - return determinable_results, reasoning, additional_information, queries, None + return determinable_results.json(), reasoning, additional_information, queries, None # Make the prediction messages = [ @@ -807,13 +808,13 @@ def run(**kwargs) -> Tuple[Optional[str], Optional[Dict[str, Any]], Any]: model=engine, ) return ( - results, + results.json(), reasoning, additional_information, queries, counter_callback, ) - return results, reasoning, additional_information, queries, None + return results.json(), reasoning, additional_information, queries, None except Exception as e: return None, None, None, None, e From 3ec10c09a23a7050ebb3a3ae9eed06395ea0cdb0 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Fri, 8 Mar 2024 14:35:57 +0000 Subject: [PATCH 4/4] fix hashes --- .../napthaai/customs/resolve_market_reasoning/component.yaml | 2 +- packages/packages.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/napthaai/customs/resolve_market_reasoning/component.yaml b/packages/napthaai/customs/resolve_market_reasoning/component.yaml index 716fb2ff..ee01f90c 100644 --- a/packages/napthaai/customs/resolve_market_reasoning/component.yaml +++ b/packages/napthaai/customs/resolve_market_reasoning/component.yaml @@ -7,7 +7,7 @@ license: Apache-2.0 aea_version: '>=1.0.0, <2.0.0' fingerprint: __init__.py: bafybeib36ew6vbztldut5xayk5553rylrq7yv4cpqyhwc5ktvd4cx67vwu - resolve_market_reasoning.py: bafybeia55z2cdeqfa2ca7p6kptz6qsyjoj5x4vmrtyt4ju7oau333i6cya + resolve_market_reasoning.py: bafybeibwboa2jc64lsnibssb5wjae75dre67sybnvr645ivzkfr4nin36y fingerprint_ignore_patterns: [] entry_point: resolve_market_reasoning.py callable: run diff --git a/packages/packages.json b/packages/packages.json index 483b0987..0d524960 100644 --- a/packages/packages.json +++ b/packages/packages.json @@ -12,7 +12,7 @@ "custom/psouranis/optimization_by_prompting/0.1.0": "bafybeibgfovquvu3gg4o7y6ud2sr536v45o2dj4shqobc7bqzzdyjbinei", "custom/nickcom007/sme_generation_request/0.1.0": "bafybeihfl4663yjkxltidavnxou7rumld5wzcr43uw23bpvb3ivefsjhv4", "custom/nickcom007/prediction_request_sme/0.1.0": "bafybeicfenfjer2kvmnstul4bzxf7xfiupbipkjzattjsieaapdflnsyni", - "custom/napthaai/resolve_market_reasoning/0.1.0": "bafybeialxdi5btf2kx4auce5sdpjzd2rea65v2l4ivmr4j5ignpum7vs54", + "custom/napthaai/resolve_market_reasoning/0.1.0": "bafybeibxt2c654xfcrw24m7wiuhlsawfszogybkadngmun3lfghmbwqbpe", "custom/napthaai/prediction_request_rag/0.1.0": "bafybeihdywbiaywvia32mpdos7e4pvcnmsq4ij65otctidbd6rjf3j6tqu", "protocol/valory/acn_data_share/0.1.0": "bafybeih5ydonnvrwvy2ygfqgfabkr47s4yw3uqxztmwyfprulwfsoe7ipq", "protocol/valory/websocket_client/0.1.0": "bafybeih43mnztdv3v2hetr2k3gezg7d3yj4ur7cxdvcyaqhg65e52s5sf4",