Merge pull request #186 from valory-xyz/feat/resolve_market_answers

add answers for invalid and not enough info to resolve market

packages/napthaai/customs/resolve_market_reasoning/component.yaml

@@ -7,7 +7,7 @@ license: Apache-2.0
 aea_version: '>=1.0.0, <2.0.0'
 fingerprint:
   __init__.py: bafybeib36ew6vbztldut5xayk5553rylrq7yv4cpqyhwc5ktvd4cx67vwu
-  resolve_market_reasoning.py: bafybeickfol4s4kilpxoanphodizenft4ldhuuuieewyvfx5gymig3x644
+  resolve_market_reasoning.py: bafybeibwboa2jc64lsnibssb5wjae75dre67sybnvr645ivzkfr4nin36y
 fingerprint_ignore_patterns: []
 entry_point: resolve_market_reasoning.py
 callable: run

packages/napthaai/customs/resolve_market_reasoning/resolve_market_reasoning.py

@@ -27,6 +27,7 @@
 from pydantic import BaseModel, Field
 from docstring_parser import parse
 import tiktoken
+import json
 from openai import OpenAI
 import numpy as np
 import faiss
@@ -158,6 +159,12 @@ class Date(OpenAISchema):
 class Results(OpenAISchema):
     has_occurred: bool = Field(..., description="Whether the event has occurred.")
 
+class Valid(OpenAISchema):
+    is_valid: bool = Field(..., description="Whether the question is valid.")
+    reason: Optional[str] = Field(..., description="Reason that the question is invalid.")
+
+class Determinable(OpenAISchema):
+    is_determinable: bool = Field(..., description="Whether it is possible to answer the question based on the information provided and reasoning.")
 
 class Document(BaseModel):
     text: str
@@ -261,6 +268,45 @@ class Document(BaseModel):
 ```
 """
 
+VALID_PROMPT = """
+* You are an expert data analyst. 
+* You are provided with a question about an event (submitted to a prediction market) under "USER_PROMPT" delimited by three backticks.
+* Your task is to determine whether the question is valid.
+* You are provided with rules that determine whether a question is invalid (as well as examples) under the label "RULES".
+* Your response should only be a function call with the information about whether a question is valid and reason as arguments.
+
+RULES
+* Questions with relative dates should be marked as invalid. E.g. Invalid: Who will be the president of the United States in 6 months? (“in 6 months depends on the current time”).
+* Questions about moral values and not facts should be marked as invalid. E.g. Invalid: “Is it ethical to eat meat?”.
+* Questions in which none of the answers are valid should be marked as invalid. E.g. Invalid: “What is the result of 1+1?” with the outcomes “0” and “1”.
+* Questions in which multiple answers are valid should be marked as invalid. E.g. Invalid: “Who will be the Time person of the year 1937?” with answers “Chiang Kai-shek” and “Soong Mei-ling” (they got the prize jointly).
+
+USER_PROMPT:
+```
+{user_prompt}
+```
+"""
+
+DETERMINABLE_PROMPT = """
+* You are an expert data analyst. 
+* You are provided with a question about an event (submitted to a prediction market) under "USER_PROMPT" delimited by three backticks.
+* You are provided with a colleague's reasoning as to whether the event occurred based on online research under the label "REASONING" delimited by three backticks.
+* Your task is to determine whether it is possible to answer whether the event actually happened before the date based on the content of this reasoning. 
+* The answer that you give should reflect the opinion in the reasoning field.
+* Your response should only be a function call with the information about whether a question is valid and reason as arguments.
+
+USER_PROMPT:
+```
+{user_prompt}
+```
+
+REASONING:
+```
+{reasoning}
+```
+
+"""
+
 SYSTEM_PROMPT = """You are a world class algorithm for generating structured output from a given input."""
 
 
@@ -628,6 +674,34 @@ def run(**kwargs) -> Tuple[Optional[str], Optional[Dict[str, Any]], Any]:
 
         engine = TOOL_TO_ENGINE[tool]
 
+        # Check if question is valid
+        messages = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": VALID_PROMPT.format(
+                    user_prompt=prompt
+                ),
+            },
+        ]
+
+        response_valid = client.chat.completions.create(
+            model=engine,
+            messages=messages,
+            temperature=DEFAULT_OPENAI_SETTINGS["temperature"],
+            max_tokens=DEFAULT_OPENAI_SETTINGS["max_tokens"],
+            n=1,
+            timeout=150,
+            stop=None,
+            functions=[Valid.openai_schema],
+        )
+
+        valid_results = Valid.from_response(response_valid)
+        print(f"Valid: {valid_results}")
+
+        if not valid_results.is_valid:
+            return valid_results.json(), None, None, None, None
+
         try:
             (
                 additional_information,
@@ -672,6 +746,34 @@ def run(**kwargs) -> Tuple[Optional[str], Optional[Dict[str, Any]], Any]:
 
             reasoning = response_reasoning.choices[0].message.content
 
+            # Check if question is determinable
+            messages = [
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": DETERMINABLE_PROMPT.format(
+                        user_prompt=prompt, reasoning=reasoning
+                    ),
+                },
+            ]
+
+            response_determinable = client.chat.completions.create(
+                model=engine,
+                messages=messages,
+                temperature=DEFAULT_OPENAI_SETTINGS["temperature"],
+                max_tokens=DEFAULT_OPENAI_SETTINGS["max_tokens"],
+                n=1,
+                timeout=150,
+                stop=None,
+                functions=[Determinable.openai_schema],
+            )
+
+            determinable_results = Determinable.from_response(response_determinable)
+            print(f"Determinable: {determinable_results}")
+
+            if not determinable_results.is_determinable:
+                return determinable_results.json(), reasoning, additional_information, queries, None
+
             # Make the prediction
             messages = [
                 {"role": "system", "content": SYSTEM_PROMPT},
@@ -706,13 +808,13 @@ def run(**kwargs) -> Tuple[Optional[str], Optional[Dict[str, Any]], Any]:
                     model=engine,
                 )
                 return (
-                    results,
+                    results.json(),
                     reasoning,
                     additional_information,
                     queries,
                     counter_callback,
                 )
-            return results, reasoning, additional_information, queries, None
+            return results.json(), reasoning, additional_information, queries, None
 
         except Exception as e:
             return None, None, None, None, e

packages/packages.json

@@ -12,7 +12,7 @@
         "custom/psouranis/optimization_by_prompting/0.1.0": "bafybeibgfovquvu3gg4o7y6ud2sr536v45o2dj4shqobc7bqzzdyjbinei",
         "custom/nickcom007/sme_generation_request/0.1.0": "bafybeihfl4663yjkxltidavnxou7rumld5wzcr43uw23bpvb3ivefsjhv4",
         "custom/nickcom007/prediction_request_sme/0.1.0": "bafybeicfenfjer2kvmnstul4bzxf7xfiupbipkjzattjsieaapdflnsyni",
-        "custom/napthaai/resolve_market_reasoning/0.1.0": "bafybeifkg7l42txwa2cwvzlhxjvrtiupskh2h3ojlh4i275wksxoqxhfp4",
+        "custom/napthaai/resolve_market_reasoning/0.1.0": "bafybeibxt2c654xfcrw24m7wiuhlsawfszogybkadngmun3lfghmbwqbpe",
         "custom/napthaai/prediction_request_rag/0.1.0": "bafybeihdywbiaywvia32mpdos7e4pvcnmsq4ij65otctidbd6rjf3j6tqu",
         "protocol/valory/acn_data_share/0.1.0": "bafybeih5ydonnvrwvy2ygfqgfabkr47s4yw3uqxztmwyfprulwfsoe7ipq",
         "protocol/valory/websocket_client/0.1.0": "bafybeih43mnztdv3v2hetr2k3gezg7d3yj4ur7cxdvcyaqhg65e52s5sf4",