From 992e43d8a70a9cb645a1a2cdecfab27953be1392 Mon Sep 17 00:00:00 2001 From: Evan Griffiths <56087052+evangriffiths@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:10:43 +0100 Subject: [PATCH] Add deployment for prediction-request-rag mech (#110) --- poetry.lock | 58 ++++++++++++++++++- .../agents/mech_agent/deploy.py | 6 ++ prediction_market_agent/run_agent.py | 3 + prediction_market_agent/tools/mech/utils.py | 14 +++++ pyproject.toml | 14 +++-- 5 files changed, 88 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index e36200d5..32ca11db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1956,6 +1956,44 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "faiss-cpu" +version = "1.8.0" +description = "A library for efficient similarity search and clustering of dense vectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "faiss-cpu-1.8.0.tar.gz", hash = "sha256:3ee1549491728f37b65267c192a94661a907154a8ae0546ad50a564b8be0d82e"}, + {file = "faiss_cpu-1.8.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:134a064c7411acf7d1d863173a9d2605c5a59bd573639ab39a5ded5ca983b1b2"}, + {file = "faiss_cpu-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ba8e6202d561ac57394c9d691ff17f8fa6eb9a077913a993fce0a154ec0176f1"}, + {file = "faiss_cpu-1.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a66e9fa7b70556a39681f06e0652f4124c8ddb0a1924afe4f0e40b6924dc845b"}, + {file = "faiss_cpu-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51aaef5a1255d0ea88ea7e52a2415f98c5dd2dd9cec10348d55136541eeec99f"}, + {file = "faiss_cpu-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:38152761242870ec7019e0397cbd0ed0b0716562029ce41a71bb38448bd6d5bc"}, + {file = "faiss_cpu-1.8.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c9e6ad94b86626be1a0faff3e53c4ca169eba88aa156d7e90c5a2e9ba30558fb"}, + {file = "faiss_cpu-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4601dbd81733bf1bc3bff690aac981289fb386dc8e60d0c4eec8a37ba6856d20"}, + {file = "faiss_cpu-1.8.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa943d3b5e8c5c77cdd629d9c3c6f78d7da616e586fdd1b94aecbf2e5fa9ba06"}, + {file = "faiss_cpu-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b644b366c3b239b34fa3e08bf65bfc78a24eda1e1ea5b2b6d9be3e8fc73d8179"}, + {file = "faiss_cpu-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:f85ecf3514850f93985be238351f5a70736133cfae784b372640aa17c6343a1b"}, + {file = "faiss_cpu-1.8.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:61abc0129a357ac00f17f5167f14dff41480de2cc852f306c3d4cd36b893ccbd"}, + {file = "faiss_cpu-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b788186d6eb94e6333e1aa8bb6c84b66e967458ecdd1cee22e16f04c43ee674c"}, + {file = "faiss_cpu-1.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5658d90a202c62e4a69c5b065785e9ddcaf6986cb395c16afed8dbe4c58c31a2"}, + {file = "faiss_cpu-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d460a372efce547e53d3c47d2c2a8a90b186ad245969048c10c1d7a1e5cf21b"}, + {file = "faiss_cpu-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:9e6520324f0a6764dd267b3c32c76958bf2b1ec36752950f6fab31a7295980a0"}, + {file = "faiss_cpu-1.8.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:fc44be179d5b7f690484ef0d0caf817fea2698a5275a0c7fb6cbf406e5b2e4d1"}, + {file = "faiss_cpu-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bbd6f0bc2e1424a12dc7e19d2cc95b53124867966b21110d26f909227e7ed1f1"}, + {file = "faiss_cpu-1.8.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06e7add0c8a06ce8fb0443c38fcaf49c45fb74527ea633b819e56452608e64f5"}, + {file = "faiss_cpu-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b864e23c1817fa6cfe9bbec096fd7140d596002934f71aa89b196ffb1b9cd846"}, + {file = "faiss_cpu-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:655433755845adbb6f0961e2f8980703640cb9faa96f1cd1ea190252149e0d0a"}, + {file = "faiss_cpu-1.8.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:e81fc376a3bcda213ffb395dda1018c953ce927c587731ad582f4e6c2b225363"}, + {file = "faiss_cpu-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8c6fa6b7eaf558307b4ab118a236e8d1da79a8685222928e4dd52e277dba144a"}, + {file = "faiss_cpu-1.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:652f6812ef2e8b0f9b18209828c590bc618aca82e7f1c1b1888f52928258e406"}, + {file = "faiss_cpu-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:304da4e0d19044374b63a5b6467028572eac4bd3f32bc9e8783d800a03fb1f02"}, + {file = "faiss_cpu-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:cb475d3f25f08c97ac64dfe026f113e2aeb9829b206b3b046256c3b40dd7eb62"}, +] + +[package.dependencies] +numpy = "*" + [[package]] name = "fastapi" version = "0.110.2" @@ -5912,6 +5950,24 @@ files = [ [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pypdf2" +version = "3.0.1" +description = "A pure-python PDF library capable of splitting, merging, cropping, and transforming PDF files" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyPDF2-3.0.1.tar.gz", hash = "sha256:a74408f69ba6271f71b9352ef4ed03dc53a31aa404d29b5d31f53bfecfee1440"}, + {file = "pypdf2-3.0.1-py3-none-any.whl", hash = "sha256:d16e4205cfee272fbdc0568b68d82be796540b1537508cef59388f839c191928"}, +] + +[package.extras] +crypto = ["PyCryptodome"] +dev = ["black", "flit", "pip-tools", "pre-commit (<2.18.0)", "pytest-cov", "wheel"] +docs = ["myst_parser", "sphinx", "sphinx_rtd_theme"] +full = ["Pillow", "PyCryptodome"] +image = ["Pillow"] + [[package]] name = "pypika" version = "0.48.9" @@ -8415,4 +8471,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.10.0" -content-hash = "6d169b507d4904914907f2277e9a42b544ba3b3725d0fc7a9d127fce5a1d25ee" +content-hash = "1050e306c29d1832fbfaf45d16723155c61cf035f36f6bd1cb1f0c00dcf3a61f" diff --git a/prediction_market_agent/agents/mech_agent/deploy.py b/prediction_market_agent/agents/mech_agent/deploy.py index c7f729b6..339747df 100644 --- a/prediction_market_agent/agents/mech_agent/deploy.py +++ b/prediction_market_agent/agents/mech_agent/deploy.py @@ -60,3 +60,9 @@ class DeployablePredictionOfflineSMEAgent(DeployableMechAgentBase): def load(self) -> None: self.local = True self.tool = MechTool.PREDICTION_OFFLINE_SME + + +class DeployablePredictionRequestRAGAgent(DeployableMechAgentBase): + def load(self) -> None: + self.local = True + self.tool = MechTool.PREDICTION_REQUEST_RAG diff --git a/prediction_market_agent/run_agent.py b/prediction_market_agent/run_agent.py index dd7ef33b..4d1400c3 100644 --- a/prediction_market_agent/run_agent.py +++ b/prediction_market_agent/run_agent.py @@ -20,6 +20,7 @@ DeployablePredictionOfflineSMEAgent, DeployablePredictionOnlineAgent, DeployablePredictionOnlineSMEAgent, + DeployablePredictionRequestRAGAgent, ) from prediction_market_agent.agents.replicate_to_omen_agent.deploy import ( DeployableReplicateToOmenAgent, @@ -39,6 +40,7 @@ class RunnableAgent(str, Enum): mech_prediction_offline = "mech_prediction-offline" mech_prediction_online_sme = "mech_prediction-online-sme" mech_prediction_offline_sme = "mech_prediction-offline-sme" + mech_prediction_request_rag = "mech_prediction-request-rag" RUNNABLE_AGENTS = { @@ -50,6 +52,7 @@ class RunnableAgent(str, Enum): RunnableAgent.mech_prediction_offline: DeployablePredictionOfflineAgent, RunnableAgent.mech_prediction_online_sme: DeployablePredictionOnlineSMEAgent, RunnableAgent.mech_prediction_offline_sme: DeployablePredictionOfflineSMEAgent, + RunnableAgent.mech_prediction_request_rag: DeployablePredictionRequestRAGAgent, } diff --git a/prediction_market_agent/tools/mech/utils.py b/prediction_market_agent/tools/mech/utils.py index 76df74c6..b1b728a3 100644 --- a/prediction_market_agent/tools/mech/utils.py +++ b/prediction_market_agent/tools/mech/utils.py @@ -8,6 +8,9 @@ from prediction_market_agent_tooling.benchmark.utils import OutcomePrediction from prediction_market_agent.tools.mech.api_keys import MechAPIKeys +from prediction_market_agent.tools.mech.mech.packages.napthaai.customs.prediction_request_rag import ( + prediction_request_rag, +) from prediction_market_agent.tools.mech.mech.packages.nickcom007.customs.prediction_request_sme import ( prediction_request_sme, ) @@ -38,6 +41,7 @@ class MechTool(str, Enum): PREDICTION_OFFLINE = "prediction-offline" PREDICTION_ONLINE_SME = "prediction-online-sme" PREDICTION_OFFLINE_SME = "prediction-offline-sme" + PREDICTION_REQUEST_RAG = "prediction-request-rag" def mech_request(question: str, mech_tool: MechTool) -> OutcomePrediction: @@ -94,6 +98,16 @@ def mech_request_local(question: str, mech_tool: MechTool) -> OutcomePrediction: "google_engine_id": keys.google_search_engine_id.get_secret_value(), }, ) + elif mech_tool == MechTool.PREDICTION_REQUEST_RAG: + response = prediction_request_rag.run( + tool=mech_tool.value, + prompt=question, + api_keys={ + "openai": keys.openai_api_key.get_secret_value(), + "google_api_key": keys.google_search_api_key.get_secret_value(), + "google_engine_id": keys.google_search_engine_id.get_secret_value(), + }, + ) else: raise ValueError(f"Mech type '{mech_tool}' not supported") diff --git a/pyproject.toml b/pyproject.toml index 8bc19ba2..d8941ed8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,12 +42,14 @@ isort = "^5.13.2" markdownify = "^0.11.6" tavily-python = "^0.3.1" microchain-python = "^0.3.5" -setuptools = "^69.5.1" -jsonschema = "^4.3.3" -chromadb = "^0.4.24" -spacy = "^3.7.4" -readability-lxml = "^0.8.1" -lxml = {extras = ["html-clean"], version = "^5.2.1"} +setuptools = "^69.5.1" # TODO remove with https://github.com/gnosis/prediction-market-agent/issues/97 +jsonschema = "^4.3.3" # TODO remove with https://github.com/gnosis/prediction-market-agent/issues/97 +chromadb = "^0.4.24" # TODO remove with https://github.com/gnosis/prediction-market-agent/issues/97 +spacy = "^3.7.4" # TODO remove with https://github.com/gnosis/prediction-market-agent/issues/97 +readability-lxml = "^0.8.1" # TODO remove with https://github.com/gnosis/prediction-market-agent/issues/97 +lxml = {extras = ["html-clean"], version = "^5.2.1"} # TODO remove with https://github.com/gnosis/prediction-market-agent/issues/97 +pypdf2 = "^3.0.1" # TODO remove with https://github.com/gnosis/prediction-market-agent/issues/97 +faiss-cpu = "^1.8.0" # TODO remove with https://github.com/gnosis/prediction-market-agent/issues/97 [build-system] requires = ["poetry-core"]