Skip to content

Commit

Permalink
Fix few flake8 issues;
Browse files Browse the repository at this point in the history
  • Loading branch information
@ranjan-stha
ranjan-stha committed Dec 5, 2024
1 parent 4066e91 commit 6821ac0
Show file tree
Hide file tree
Showing 8 changed files with 150 additions and 170 deletions.
226 changes: 109 additions & 117 deletions analysis_module/mock_templates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -612,42 +612,42 @@
# this mdoel prediction refer to the framework_id: 1623
# entry_id: 510021, project_id: 2587 (2021 IFRC ESSN Turkey)
"model_tags": {
"element1":{
"pillar-0":{
"subpillar-4":{
"sector-1":[
"element1": {
"pillar-0": {
"subpillar-4": {
"sector-1": [
"t31unid23fntmwrb"
],
"sector-4":[
],
"sector-4": [
"subsector-1"
]
},
"subpillar-3":{
"sector-4":[
"subsector-1"
]
}
]
},
"pillar-1":{
"subpillar-1":{
"sector-4":[
"subpillar-3": {
"sector-4": [
"subsector-1"
]
}
]
}
},
"overview-matrix1dWidget-d48u7z4yohwuu7zg":{
"8lowwhswgb5j9f5s":{
"gjsbosuej330kl45":True,
"iwsqjtrs2u5z8qgk":True
}
},
"element0":{
"pillar-3":{
"subpillar-21":True,
"subpillar-20":True
"pillar-1": {
"subpillar-1": {
"sector-4": [
"subsector-1"
]
}
}
},
"overview-matrix1dWidget-d48u7z4yohwuu7zg": {
"8lowwhswgb5j9f5s": {
"gjsbosuej330kl45": True,
"iwsqjtrs2u5z8qgk": True
}
},
"element0": {
"pillar-3": {
"subpillar-21": True,
"subpillar-20": True
}
}
},
"geolocations": [
{
Expand Down Expand Up @@ -1809,103 +1809,95 @@
]
}

MOCK_ENTRY_EXTRACTION_LLM = {
MOCK_ENTRY_EXTRACTION_LLM = {
# this model prediction refer to the framework_id: 1623
# lead_id 67027, url: 'https://reliefweb.int/sites/reliefweb.int/files/resources/UNHCR-Turkey-Operational-Update-October-2019.pdf'
# lead_id 67027, url: 'https://reliefweb.int/sites/reliefweb.int/files/resources/UNHCR-Turkey-Operational-Update-October-2019.pdf' # noqa
# project_id: 2587 (2021 IFRC ESSN Turkey)
"client_id": "entry-classification-llm-client-6000",
"metadata": {
"metadata": {
"total_pages": 10,
"total_words_count": 5876
},
"blocks": [
{
"type":"text",
"text":"4 million Refugees and asylum-seekers in Turkey including over 3.6 million Syrian nationals and close to 400,000 registered refugees and asylum-seekers of other nationalities. Over 98% of Syrian refugees live across Turkey in 81 provinces",
"page":0,
"textOrder":2,
"relevant":True,
"prediction_status":True,
"classification":{
"element1":{
"pillar-0":{
"o9kyhltzmplk0a1k":{
"sector-9":[

]
}
},
"46bg6n1o50obgx77":{
"v2kfnyjbn41vv46j":{
"sector-9":[

]
}
}
},
"overview-matrix1dWidget-d48u7z4yohwuu7zg":{
"8lowwhswgb5j9f5s":{
"qycslaise1s014vm":True,
"7wps5hbnemt59dv9":True
}
},
"element0":{
"pillar-0":{
"nxjm8rsprb9fu2wq":True
"blocks": [{
"type": "text",
"text": "4 million Refugees and asylum-seekers in Turkey including over 3.6 million Syrian nationals and close to 400,000 registered refugees and asylum-seekers of other nationalities. Over 98% of Syrian refugees live across Turkey in 81 provinces",
"page": 0,
"textOrder": 2,
"relevant": True,
"prediction_status": True,
"classification": {
"element1": {
"pillar-0": {
"o9kyhltzmplk0a1k": {
"sector-9": []
}
},
"46bg6n1o50obgx77": {
"v2kfnyjbn41vv46j": {
"sector-9": []
}
}
},
"kyiciutprwct1vph":{
"4ftpwnssu2ugeekk":True
}
}
},
"geolocations": [
{
"entity": "Niger",
"meta": {
"offset_start": 88,
"offset_end": 94,
"latitude": -10,
"longitude": -55
"overview-matrix1dWidget-d48u7z4yohwuu7zg": {
"8lowwhswgb5j9f5s": {
"qycslaise1s014vm": True,
"7wps5hbnemt59dv9": True
}
},
"element0": {
"pillar-0": {
"nxjm8rsprb9fu2wq": True
},
"kyiciutprwct1vph": {
"4ftpwnssu2ugeekk": True
}
}
},
{
"entity": "Nigeria",
"meta": {
"offset_start": 183,
"offset_end": 191,
"latitude": None,
"longitude": None
}
}
],
},
{
"type":"text",
"text":"9,700 Refugees departed for resettlement in 2019 as of end of October, over 78 per cent of whom are Syrians",
"page":0,
"textOrder":3,
"relevant":True,
"prediction_status":True,
"classification":{
"element1":{
"pillar-0":{
"o9kyhltzmplk0a1k":{
"sector-9":[

]
}
}
},
"overview-matrix1dWidget-d48u7z4yohwuu7zg":{
"8lowwhswgb5j9f5s":{True
"geolocations": [
{
"entity": "Niger",
"meta": {
"offset_start": 88,
"offset_end": 94,
"latitude": -10,
"longitude": -55
}
},
{
"entity": "Nigeria",
"meta": {
"offset_start": 183,
"offset_end": 191,
"latitude": None,
"longitude": None
}
}
},
"element0":{
"kyiciutprwct1vph":{
"4ftpwnssu2ugeekk":True
],
},
{
"type": "text",
"text": "9,700 Refugees departed for resettlement in 2019 as of end of October, over 78 per cent of whom are Syrians",
"page": 0,
"textOrder": 3,
"relevant": True,
"prediction_status": True,
"classification": {
"element1": {
"pillar-0": {
"o9kyhltzmplk0a1k": {
"sector-9": []
}
}
},
"overview-matrix1dWidget-d48u7z4yohwuu7zg": {
"8lowwhswgb5j9f5s": True
},
"element0": {
"kyiciutprwct1vph": {
"4ftpwnssu2ugeekk": True
}
}
}
}
}
]
}
}
}
]
}
9 changes: 6 additions & 3 deletions analysis_module/mockserver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@

from core.models import NLPRequest
from core_server.settings import ENDPOINT_NAME
from .mock_templates import (MOCK_ENTRY_CLASSIFICATION,
from .mock_templates import (MOCK_ENTRY_CLASSIFICATION,
MOCK_ENTRY_CLASSIFICATION_LLM,
MOCK_ENTRY_CLASSIFICATION_FORMATTED,
MOCK_ENTRY_CLASSIFICATION_FORMATTED,
MOCK_ENTRY_EXTRACTION_LLM,
MOCK_GEOLOCATION) # noqa
MOCK_GEOLOCATION) # noqa
from .utils import send_callback_url_request


Expand Down Expand Up @@ -500,12 +500,14 @@ def process_entry_extraction_mock(body) -> Any:
except Exception:
logger.error("Could not send data to callback url", exc_info=True)


def entry_extraction_llm_mock(body) -> Any:
process_entry_extraction_llm_mock.apply_async(
args=(body,), countdown=2
) # Trigger task after 2 seconds
return json.dumps({"status": "Successfully received the request."}), 200


@shared_task
def process_entry_extraction_llm_mock(body) -> Any:
documents = body.get("documents") or []
Expand Down Expand Up @@ -554,6 +556,7 @@ def process_entry_extraction_llm_mock(body) -> Any:
except Exception:
logger.error("Could not send data to callback url", exc_info=True)


def entry_classification_mock(body) -> Any:
process_entry_classification_mock.apply_async(
args=(body,), countdown=2
Expand Down
7 changes: 3 additions & 4 deletions analysis_module/serializers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,6 @@ def to_representation(self, value):


class EntryExtractionSerializer(serializers.Serializer):

documents = DocumentEntryExtractionUnionField()
callback_url = serializers.CharField()
request_type = serializers.ChoiceField(
Expand All @@ -156,14 +155,14 @@ class EntryExtractionSerializer(serializers.Serializer):
)
mock = serializers.BooleanField(default=False)

class EntryExtractionSerializerLLM(serializers.Serializer):

class EntryExtractionSerializerLLM(serializers.Serializer):
documents = DocumentEntryExtractionUnionField()
af_id = serializers.IntegerField()
af_id = serializers.IntegerField()
project_id = serializers.IntegerField()
callback_url = serializers.CharField()
request_type = serializers.ChoiceField(
choices=ExtractionRequestTypeChoices,
default=ExtractionRequestTypeChoices.USER,
)
mock = serializers.BooleanField(default=False)
mock = serializers.BooleanField(default=False)
3 changes: 1 addition & 2 deletions analysis_module/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,6 @@ def get_geolocations(excerpts: List[str], req_timeout: int = 60):

@shared_task
def send_classification_tags(nlp_request_id: int, version: str = "v1"):

nlp_request = NLPRequest.objects.get(pk=nlp_request_id)
predictor = ModelTagsPrediction()
entries_dict = nlp_request.request_params["entries"]
Expand Down Expand Up @@ -221,7 +220,6 @@ def send_classification_tags(nlp_request_id: int, version: str = "v1"):

@shared_task
def send_classification_tags_llm(nlp_request_id: int, version: str = "v1"):

nlp_request = NLPRequest.objects.get(pk=nlp_request_id)
predictor = LLMTagsPrediction(analysis_framework_id=nlp_request.request_params['af_id'])
entries_dict = nlp_request.request_params["entries"]
Expand Down Expand Up @@ -254,6 +252,7 @@ def send_classification_tags_llm(nlp_request_id: int, version: str = "v1"):
except Exception:
logger.error("Could not send http request on callback url : {callback_url}", exc_info=True)


def send_ecs_http_request(nlp_request: NLPRequest):
ecs_id_param_name = get_ecs_id_param_name(nlp_request.type)
url = get_ecs_url(nlp_request.type)
Expand Down
2 changes: 1 addition & 1 deletion core/tasks/queries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,4 +220,4 @@
FROM geo_geoarea
"""

af_widget_by_id = "SELECT * from analysis_framework_widget ll WHERE ll.analysis_framework_id={}"
af_widget_by_id = "SELECT * from analysis_framework_widget ll WHERE ll.analysis_framework_id={}"
2 changes: 1 addition & 1 deletion core_server/env.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
RELIABILITY_MODEL_ID=str,
RELIABILITY_MODEL_VERSION=str,

# LLM
# LLM
OPENAI_API_KEY=str,
OPENAI_MAIN_MODEL=(str, "gpt-4o"),
OPENAI_SMALL_MODEL=(str, "gpt-4o-mini"),
Expand Down
Loading

0 comments on commit 6821ac0

Please sign in to comment.