Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: enable filtering #65

Merged
merged 5 commits into from
Sep 23, 2024
Merged

refactor: enable filtering #65

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
651326d
refactor: enable filtering
mehmetcanay Sep 3, 2024
08d20cd
fix: model selection
mehmetcanay Sep 16, 2024
7e33846
Merge branch 'main' into refactor-endpoint
mehmetcanay Sep 16, 2024
b732aa1
refactor: remove token using models
mehmetcanay Sep 17, 2024
ff467ea
Merge branch 'refactor-endpoint' of https://github.com/SCAI-BIO/index…
mehmetcanay Sep 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 44 additions & 34 deletions api/routes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@

import uvicorn
from datastew import DataDictionarySource
from datastew.embedding import MPNetAdapter
from datastew.embedding import GPT4Adapter, MPNetAdapter
from datastew.process.ols import OLSTerminologyImportTask
from datastew.repository import WeaviateRepository
from datastew.repository.model import Terminology, Concept, Mapping
from datastew.repository.model import Concept, Mapping, Terminology
from datastew.visualisation import get_plot_for_current_database_state
from fastapi import FastAPI, HTTPException, File, UploadFile
from fastapi import FastAPI, File, HTTPException, UploadFile
from starlette.background import BackgroundTasks
from starlette.middleware.cors import CORSMiddleware
from starlette.responses import RedirectResponse, HTMLResponse
from starlette.responses import HTMLResponse, RedirectResponse

app = FastAPI(
title="INDEX",
Expand Down Expand Up @@ -190,39 +190,47 @@ async def get_closest_mappings_for_text(text: str,
terminology_name: str = "SNOMED CT",
model: str = "sentence-transformers/all-mpnet-base-v2",
limit: int = 5):
embedding_model = MPNetAdapter(model)
embedding = embedding_model.get_embedding(text).tolist()
closest_mappings = repository.get_terminology_and_model_specific_closest_mappings(embedding, terminology_name,
model, limit)
mappings = []
for mapping, similarity in closest_mappings:
concept = mapping.concept
terminology = concept.terminology
mappings.append({
"concept": {
"id": concept.concept_identifier,
"name": concept.pref_label,
"terminology": {
"id": terminology.id,
"name": terminology.name
}
},
"text": mapping.text,
"similarity": similarity
})
try:
embedding_model = MPNetAdapter(model)
embedding = embedding_model.get_embedding(text).tolist()
closest_mappings = repository.get_terminology_and_model_specific_closest_mappings(embedding, terminology_name, model, limit)
mappings = []
for mapping, similarity in closest_mappings:
concept = mapping.concept
terminology = concept.terminology
mappings.append({
"concept": {
"id": concept.concept_identifier,
"name": concept.pref_label,
"terminology": {
"id": terminology.id,
"name": terminology.name
}
},
"text": mapping.text,
"similarity": similarity
})

return mappings
return mappings
except Exception as e:
raise HTTPException(status_code=400, detail=f"Failed to get closest mappings: {str(e)}")


# Endpoint to get mappings for a data dictionary source
@app.post("/mappings/dict", tags=["mappings"], description="Get mappings for a data dictionary source.")
async def get_closest_mappings_for_dictionary(file: UploadFile = File(...),
variable_field: str = 'variable',
description_field: str = 'description',
model: str = "sentence-transformers/all-mpnet-base-v2"):
async def get_closest_mappings_for_dictionary(file: UploadFile = File(...),
model: str = "sentence-transformers/all-mpnet-base-v2",
terminology_name: str = "SNOMED CT",
variable_field: str = "variable",
description_field: str = "description"):
try:
embedding_model = MPNetAdapter(model)
# Determine file extension and create a temporary file with the correct extension
_, file_extension = os.path.splitext(file.filename)
if file.filename is not None:
file_extension = os.path.splitext(file.filename)[1].lower()
else:
raise HTTPException(status_code=400, detail="Invalid file type. The file must have a suffix.")

with tempfile.NamedTemporaryFile(delete=False, suffix=file_extension) as tmp_file:
tmp_file.write(await file.read())
tmp_file_path = tmp_file.name
Expand All @@ -238,15 +246,17 @@ async def get_closest_mappings_for_dictionary(file: UploadFile = File(...),
description = row['description']
embedding_model = MPNetAdapter(model)
embedding = embedding_model.get_embedding(description)
closest_mappings, similarities = repository.get_closest_mappings(embedding, limit=5)
closest_mappings = repository.get_terminology_and_model_specific_closest_mappings(
embedding, terminology_name, model, limit=5
)
mappings_list = []
for mapping, similarity in zip(closest_mappings, similarities):
for mapping, similarity in closest_mappings:
concept = mapping.concept
terminology = concept.terminology
mappings_list.append({
"concept": {
"id": concept.concept_id,
"name": concept.name,
"id": concept.concept_identifier,
"name": concept.pref_label,
"terminology": {
"id": terminology.id,
"name": terminology.name
Expand Down
Loading