From 8b59afbc1e2f1f0e1148a12419209cb647383bf6 Mon Sep 17 00:00:00 2001 From: Erik M Jacobs Date: Thu, 2 Nov 2023 11:48:15 -0400 Subject: [PATCH] big refactor * moves things to tools and modules folder * adds comments for direct invocation * modifies ols to support new folder structure * changes to ignore tmptest* files for experimentation * adds kubernetes python module and sorts requirements --- .gitignore | 5 ++++- __init__.py | 0 modules/__init__.py | 0 model_context.py => modules/model_context.py | 0 .../question_validator.py | 3 ++- task_breakdown.py => modules/task_breakdown.py | 3 ++- task_performer.py => modules/task_performer.py | 3 ++- task_processor.py => modules/task_processor.py | 9 +++++---- task_rephraser.py => modules/task_rephraser.py | 7 ++++--- yes_no_classifier.py => modules/yes_no_classifier.py | 3 ++- ols.py | 6 +++--- requirements.txt | 11 ++++++----- indexer.py => tools/indexer.py | 2 +- json_tools.py => tools/json_tools.py | 0 14 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 __init__.py create mode 100644 modules/__init__.py rename model_context.py => modules/model_context.py (100%) rename question_validator.py => modules/question_validator.py (96%) rename task_breakdown.py => modules/task_breakdown.py (97%) rename task_performer.py => modules/task_performer.py (95%) rename task_processor.py => modules/task_processor.py (95%) rename task_rephraser.py => modules/task_rephraser.py (93%) rename yes_no_classifier.py => modules/yes_no_classifier.py (95%) rename indexer.py => tools/indexer.py (94%) rename json_tools.py => tools/json_tools.py (100%) diff --git a/.gitignore b/.gitignore index 7cfb1603c..8a1184c4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ __pycache__ tmp data -vector-db \ No newline at end of file +vector-db + +# ignore files that begin with tmptest_ +tmptest_* \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/model_context.py b/modules/model_context.py similarity index 100% rename from model_context.py rename to modules/model_context.py diff --git a/question_validator.py b/modules/question_validator.py similarity index 96% rename from question_validator.py rename to modules/question_validator.py index 7dbd326be..0fcc44a7c 100644 --- a/question_validator.py +++ b/modules/question_validator.py @@ -1,6 +1,6 @@ import logging import sys -from model_context import get_watsonx_predictor +from modules.model_context import get_watsonx_predictor from langchain.chains import LLMChain from langchain.prompts import PromptTemplate @@ -95,6 +95,7 @@ def validate_question(self, conversation, query, **kwargs): return response['text'] if __name__ == "__main__": + """to execute, from the repo root, use python -m modules.question_validator.py""" import argparse parser = argparse.ArgumentParser( diff --git a/task_breakdown.py b/modules/task_breakdown.py similarity index 97% rename from task_breakdown.py rename to modules/task_breakdown.py index f7e6ec9a3..0d8eba170 100644 --- a/task_breakdown.py +++ b/modules/task_breakdown.py @@ -1,6 +1,6 @@ import llama_index from llama_index import StorageContext, load_index_from_storage -from model_context import get_watsonx_context +from modules.model_context import get_watsonx_context from llama_index.prompts import Prompt, PromptTemplate import logging import sys @@ -101,6 +101,7 @@ def breakdown_tasks(self, conversation, query, **kwargs): if __name__ == "__main__": + """to execute, from the repo root, use python -m modules.task_breakdown.py""" import argparse parser = argparse.ArgumentParser( diff --git a/task_performer.py b/modules/task_performer.py similarity index 95% rename from task_performer.py rename to modules/task_performer.py index 56a4b39b2..c806cd2b9 100644 --- a/task_performer.py +++ b/modules/task_performer.py @@ -1,7 +1,7 @@ import logging import sys from string import Template -from model_context import get_watsonx_predictor +from modules.model_context import get_watsonx_predictor DEFAULT_MODEL = "ibm/granite-13b-chat-grounded-v01" @@ -67,6 +67,7 @@ def perform_task(self, conversation, task, **kwargs): if __name__ == "__main__": + """to execute, from the repo root, use python -m modules.task_performer.py""" import argparse parser = argparse.ArgumentParser(description="Perform a task") diff --git a/task_processor.py b/modules/task_processor.py similarity index 95% rename from task_processor.py rename to modules/task_processor.py index e9186a6b9..698129894 100644 --- a/task_processor.py +++ b/modules/task_processor.py @@ -1,9 +1,9 @@ import logging import sys -from model_context import get_watsonx_predictor -from yes_no_classifier import YesNoClassifier -from task_performer import TaskPerformer -from task_rephraser import TaskRephraser +from modules.model_context import get_watsonx_predictor +from modules.yes_no_classifier import YesNoClassifier +from modules.task_performer import TaskPerformer +from modules.task_rephraser import TaskRephraser from langchain.chains import LLMChain from langchain.prompts import PromptTemplate @@ -153,6 +153,7 @@ def process_tasks(self, conversation, tasklist, original_query, **kwargs): if __name__ == "__main__": + """to execute, from the repo root, use python -m modules.task_processor.py""" import argparse parser = argparse.ArgumentParser(description="Process a list of tasks") diff --git a/task_rephraser.py b/modules/task_rephraser.py similarity index 93% rename from task_rephraser.py rename to modules/task_rephraser.py index 6495d2934..6f97b7346 100644 --- a/task_rephraser.py +++ b/modules/task_rephraser.py @@ -1,8 +1,8 @@ import logging import sys -from model_context import get_watsonx_predictor -from yes_no_classifier import YesNoClassifier -from task_performer import TaskPerformer +from modules.model_context import get_watsonx_predictor +from modules.yes_no_classifier import YesNoClassifier +from modules.task_performer import TaskPerformer from langchain.chains import LLMChain from langchain.prompts import PromptTemplate @@ -85,6 +85,7 @@ def rephrase_task(self, conversation, task, original_query, **kwargs): return response['text'] if __name__ == "__main__": + """to execute, from the repo root, use python -m modules.task_rephraser.py""" import argparse parser = argparse.ArgumentParser( diff --git a/yes_no_classifier.py b/modules/yes_no_classifier.py similarity index 95% rename from yes_no_classifier.py rename to modules/yes_no_classifier.py index 5c84f99ba..078b77427 100644 --- a/yes_no_classifier.py +++ b/modules/yes_no_classifier.py @@ -1,6 +1,6 @@ import logging, sys from string import Template -from model_context import get_watsonx_predictor +from modules.model_context import get_watsonx_predictor from langchain.chains import LLMChain from langchain.prompts import PromptTemplate @@ -89,6 +89,7 @@ def classify(self, conversation, string, **kwargs): if __name__ == "__main__": + """to execute, from the repo root, use PYTHONPATH="$PYTHONPATH:`pwd`" python modules/yes_no_classifier.py""" import argparse parser = argparse.ArgumentParser(description="Process a list of tasks") diff --git a/ols.py b/ols.py index 7a6547604..3e162d472 100644 --- a/ols.py +++ b/ols.py @@ -2,8 +2,8 @@ from fastapi import FastAPI, HTTPException from pydantic import BaseModel -from task_breakdown import TaskBreakdown -from task_processor import TaskProcessor +from modules.task_breakdown import TaskBreakdown +from modules.task_processor import TaskProcessor import logging, sys, os @@ -16,7 +16,7 @@ import uuid ## internal stuff -from model_context import get_watsonx_predictor +from modules.model_context import get_watsonx_predictor instruct_model = os.getenv("INSTRUCT_MODEL", "ibm/granite-13b-instruct-v1") rag_model = os.getenv("RAG_MODEL", "ibm/granite-13b-chat-grounded-v01") diff --git a/requirements.txt b/requirements.txt index 6891ba586..4979f9bf3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ fastapi -uvicorn -llama_index -langchain -ibm_watson_machine_learning ibm-generative-ai +ibm_watson_machine_learning +kubernetes +langchain +llama_index +torch transformers -torch \ No newline at end of file +uvicorn \ No newline at end of file diff --git a/indexer.py b/tools/indexer.py similarity index 94% rename from indexer.py rename to tools/indexer.py index 4610d4aa3..52a03fe06 100644 --- a/indexer.py +++ b/tools/indexer.py @@ -2,7 +2,7 @@ import llama_index from llama_index import VectorStoreIndex, SimpleDirectoryReader from llama_index.storage.storage_context import StorageContext -from model_context import get_watsonx_context +from modules.model_context import get_watsonx_context import logging, sys, os #logging.basicConfig(stream=sys.stdout, level=logging.INFO) diff --git a/json_tools.py b/tools/json_tools.py similarity index 100% rename from json_tools.py rename to tools/json_tools.py