Skip to content

Commit

Permalink
Merge pull request #22 from KevKibe/create-delete-index-refactor
Browse files Browse the repository at this point in the history
Refactor: Move `create_index`, `delete_index`, `doc_model`, `config` sub-modules into utils module.
  • Loading branch information
KevKibe authored May 14, 2024
2 parents 15e167f + fbb200a commit 7e6ced9
Show file tree
Hide file tree
Showing 16 changed files with 46 additions and 121 deletions.
26 changes: 21 additions & 5 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Tests
name: CoherePineconeIndexer Tests

on: [pull_request]

Expand All @@ -20,14 +20,30 @@ jobs:
pip install -r requirements.txt --no-cache-dir
pip install pytest einops lion-pytorch
- name: Test with pytest
- name: Test CoherePineconeIndexer
id: cohere_test
env:
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
run: |
pytest src/tests/cohereindex_test.py
- name: Test OpenaiPineconeIndexer
id: openai_test
env:
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
run: |
pytest src/tests/openaiindex_test.py
- name: Test GooglePineconeIndexer
id: google_test
env:
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
run: |
pytest src/tests/googleindex_test.py
pytest src/tests/cohereindex_test.py

4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ cd src

```bash
# Using OpenAI
python -m _openai.create_index --pinecone_api_key "your_pinecone_api_key" --index_name "your_index_name" -
python -m utils.create_index --pinecone_api_key "your_pinecone_api_key" --index_name "your_index_name"

```

Expand All @@ -164,7 +164,7 @@ python -m _cohere.index_documents --pinecone_api_key "your_pinecone_api_key" -

```bash
# Using OpenAI
python -m _openai.delete_index --pinecone_api_key "your_pinecone_api_key" --index_name "your_index_name"
python -m utils.delete_index --pinecone_api_key "your_pinecone_api_key" --index_name "your_index_name"

```

Expand Down
12 changes: 0 additions & 12 deletions src/_cohere/config.py

This file was deleted.

2 changes: 1 addition & 1 deletion src/_cohere/doc_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from langchain.text_splitter import RecursiveCharacterTextSplitter
import tiktoken
from typing import List
from _openai.doc_model import Page
from utils.doc_model import Page
from langchain_pinecone import PineconeVectorStore
from pathlib import Path
from langchain_community.document_loaders import UnstructuredWordDocumentLoader
Expand Down
11 changes: 0 additions & 11 deletions src/_google/config.py

This file was deleted.

15 changes: 0 additions & 15 deletions src/_google/create_index.py

This file was deleted.

15 changes: 0 additions & 15 deletions src/_google/delete_index.py

This file was deleted.

6 changes: 3 additions & 3 deletions src/_google/doc_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from langchain_google_genai import GoogleGenerativeAIEmbeddings
import tiktoken
from typing import List
from _openai.doc_model import Page
from utils.doc_model import Page
import google.generativeai as genai
from pathlib import Path
from langchain_community.document_loaders import UnstructuredWordDocumentLoader
Expand All @@ -27,8 +27,8 @@ class GooglePineconeIndexer:
def __init__(
self,
index_name: str,
pinecone_api_key: str,
google_api_key: str
pinecone_api_key: str = None,
google_api_key: str = None
) -> None:
"""
Initialize the GoogleGenerativeAIEmbeddings object.
Expand Down
15 changes: 0 additions & 15 deletions src/_openai/create_index.py

This file was deleted.

15 changes: 0 additions & 15 deletions src/_openai/delete_index.py

This file was deleted.

6 changes: 3 additions & 3 deletions src/_openai/doc_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from langchain_openai import OpenAIEmbeddings
import tiktoken
from typing import List
from .doc_model import Page
from utils.doc_model import Page
from pathlib import Path
from langchain_community.document_loaders import UnstructuredWordDocumentLoader
from langchain_community.document_loaders import UnstructuredMarkdownLoader
Expand All @@ -26,8 +26,8 @@ class OpenaiPineconeIndexer:
def __init__(
self,
index_name: str,
pinecone_api_key: str,
openai_api_key: str
pinecone_api_key: str = None ,
openai_api_key: str = None
) -> None:
"""
Initialize the OpenAIPineconeIndexer object.
Expand Down
8 changes: 0 additions & 8 deletions src/_openai/doc_model.py

This file was deleted.

20 changes: 10 additions & 10 deletions src/tests/googleindex_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,16 @@ def test_03_initialize_vectorstore(self):
vectorstore = self.indexer.initialize_vectorstore(self.index_name)
self.assertIsInstance(vectorstore, PineconeVectorStore)

def test_04_retrieve_and_generate(self):
"""
Test initializing the vector store and assert its type.
"""
vector_store = self.indexer.initialize_vectorstore(self.index_name)
response = self.indexer.retrieve_and_generate(query = "give a short summary of the introduction",
vector_store= vector_store
)
print(response)
self.assertIsNotNone(response, "The retriever response should not be None.")
# def test_04_retrieve_and_generate(self):
# """
# Test initializing the vector store and assert its type.
# """
# vector_store = self.indexer.initialize_vectorstore(self.index_name)
# response = self.indexer.retrieve_and_generate(query = "give a short summary of the introduction",
# vector_store= vector_store
# )
# print(response)
# self.assertIsNotNone(response, "The retriever response should not be None.")

@patch('sys.stdout', new_callable=StringIO)
def test_05_delete_index(self, mock_stdout):
Expand Down
6 changes: 3 additions & 3 deletions src/_cohere/create_index.py → src/utils/create_index.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
from .doc_index import CoherePineconeIndexer
from _openai.doc_index import OpenaiPineconeIndexer
import argparse

def parse_args():
parser = argparse.ArgumentParser(description="Creates an Index on Pinecone.")
parser = argparse.ArgumentParser(description="Create an Index on Pinecone.")
parser.add_argument("--pinecone_api_key", type=str, help="Pinecone API key")
parser.add_argument("--index_name", type=str, help="Name of the Pinecone index")
return parser.parse_args()


if __name__ == "__main__":
args = parse_args()
pinecone_indexer = CoherePineconeIndexer(args.index_name, args.pinecone_api_key)
pinecone_indexer = OpenaiPineconeIndexer(args.index_name, args.pinecone_api_key)
pinecone_indexer.create_index()
6 changes: 3 additions & 3 deletions src/_cohere/delete_index.py → src/utils/delete_index.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
from .doc_index import CoherePineconeIndexer
from _openai.doc_index import OpenaiPineconeIndexer
import argparse

def parse_args():
parser = argparse.ArgumentParser(description="Deletes an Index on Pinecone.")
parser = argparse.ArgumentParser(description="Delete an existing Index on Pinecone.")
parser.add_argument("--pinecone_api_key", type=str, help="Pinecone API key")
parser.add_argument("--index_name", type=str, help="Name of the Pinecone index")
return parser.parse_args()


if __name__ == "__main__":
args = parse_args()
pinecone_indexer = CoherePineconeIndexer(args.index_name, args.pinecone_api_key)
pinecone_indexer = OpenaiPineconeIndexer(args.index_name, args.pinecone_api_key)
pinecone_indexer.delete_index()
File renamed without changes.

0 comments on commit 7e6ced9

Please sign in to comment.