diff --git a/.flake8 b/.flake8 index 805618e..f792b65 100644 --- a/.flake8 +++ b/.flake8 @@ -7,4 +7,3 @@ max-complexity = 10 per-file-ignores = /**/tests/*_mock_data.py: E501 **/snap_test_*.py: E501 - /**/apps.py: F401 diff --git a/content/apps.py b/content/apps.py index a067f2c..087ec17 100644 --- a/content/apps.py +++ b/content/apps.py @@ -4,6 +4,3 @@ class ContentConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "content" - - def ready(self): - import content.signals diff --git a/content/models.py b/content/models.py index 64c2860..c6c015d 100644 --- a/content/models.py +++ b/content/models.py @@ -4,6 +4,7 @@ from django.utils.translation import gettext_lazy as _ from common.models import UserResource +from content.tasks import create_embedding_for_content_task class Tag(models.Model): @@ -46,4 +47,6 @@ def save(self, *args, **kwargs): if self.pk is None: if self.document_type == self.DocumentType.TEXT: self.extracted_file = self.document_file + if self.document_status == self.DocumentStatus.PENDING: + create_embedding_for_content_task(self.id) super().save(*args, **kwargs) diff --git a/content/signals.py b/content/signals.py deleted file mode 100644 index f716e13..0000000 --- a/content/signals.py +++ /dev/null @@ -1,10 +0,0 @@ -from django.db.models.signals import post_save -from django.dispatch import receiver - -from .models import Content -from .tasks import create_embedding_for_content_task - - -@receiver(post_save, sender=Content) -def content_handler(sender, instance, created, **kwargs): - create_embedding_for_content_task.delay(instance.id) diff --git a/content/tasks.py b/content/tasks.py index dd36784..ae5f03f 100644 --- a/content/tasks.py +++ b/content/tasks.py @@ -8,10 +8,9 @@ @shared_task(blind=True) def create_embedding_for_content_task(content_id): - from .models import Content + from content.models import Content content = Content.objects.get(id=content_id) - url = settings.EMBEDDING_MODEL_URL headers = {"Content-Type": "application/json"} data = content.extracted_file.read() loader = LoaderFromText(text=data) @@ -22,15 +21,18 @@ def create_embedding_for_content_task(content_id): "name_model": settings.EMBEDDING_MODEL_NAME, "texts": [split_docs[i].page_content for i in range(len(split_docs))], } - response = requests.post(url, headers=headers, json=payload) + response = requests.post(settings.EMBEDDING_MODEL_URL, headers=headers, json=payload) metadata = [ {"source": "plain-text", "page_content": split_docs[i].page_content, "uuid": content.content_id} for i in range(len(split_docs)) ] if response.status_code == 200: - db = QdrantDatabase(host="qdrant", port=settings.QDRANT_DB_PORT, collection_name=settings.QDRANT_DB_COLLECTION_NAME) + db = QdrantDatabase( + host=settings.QDRANT_DB_HOST, port=settings.QDRANT_DB_PORT, collection_name=settings.QDRANT_DB_COLLECTION_NAME + ) db.set_collection() db.store_data(zip(response.json(), metadata)) content.document_status = Content.DocumentStatus.ADDED_TO_VECTOR else: content.document_status = Content.DocumentStatus.FAILURE + content.save() diff --git a/content/urls.py b/content/urls.py deleted file mode 100644 index 22a1c75..0000000 --- a/content/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -from django.urls import path - -from .views import chat - -urlpatterns = [ - path("", chat, name="chat"), -] diff --git a/content/views.py b/content/views.py index 61534cf..cdeed8e 100644 --- a/content/views.py +++ b/content/views.py @@ -1,5 +1,6 @@ # Create your views here. from rest_framework.decorators import api_view +from rest_framework.generics import GenericAPIView from rest_framework.response import Response from chatbotcore.llm import OllamaHandler @@ -14,3 +15,14 @@ def chat(request): result = data.execute_chain(request.data["query"]) return Response(result) return Response(serializer.errors) + + +class UserQuery(GenericAPIView): + llm = OllamaHandler() + + def post(self, request, *arg, **kwargs): + serializer = UserQuerySerializer(data=request.data) + if serializer.is_valid(): + result = self.llm.execute_chain(request.data["query"]) + return Response(result) + return Response(serializer.errors, 422) diff --git a/docker-compose.yml b/docker-compose.yml index cd163fc..f2ac6f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ x-server: &base_server_setup APP_ENVIRONMENT: ${APP_ENVIRONMENT:-development} APP_TYPE: web DJANGO_DEBUG: ${DJANGO_DEBUG:-true} - DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY?error} DJANGO_TIME_ZONE: ${DJANGO_TIME_ZONE:-Asia/Kathmandu} # -- Domain configurations DJANGO_ALLOWED_HOSTS: ${DJANGO_ALLOWED_HOSTS:-*} @@ -84,7 +84,7 @@ services: # https://github.com/qdrant/qdrant/blob/master/config/config.yaml QDRANT__SERVICE__HOST: 0.0.0.0 QDRANT__SERVICE__HTTP_PORT: 6333 - + web: <<: *base_server_setup command: bash -c 'python manage.py runserver 0.0.0.0:8001' @@ -99,10 +99,6 @@ services: worker: <<: *base_server_setup command: celery -A main worker --loglevel=info - depends_on: - - redis - - web - volumes: postgres-data16: diff --git a/main/urls.py b/main/urls.py index 35b68f6..89a0cf0 100644 --- a/main/urls.py +++ b/main/urls.py @@ -18,12 +18,11 @@ from django.conf import settings from django.conf.urls.static import static from django.contrib import admin -from django.urls import include, path +from django.urls import path -urlpatterns = [ - path("admin/", admin.site.urls), - path("chat_message", include("content.urls")), -] +from content.views import UserQuery + +urlpatterns = [path("admin/", admin.site.urls), path("chat_message", UserQuery.as_view())] if settings.DEBUG: # Static and media file URLs