Skip to content

Commit

Permalink
Merge pull request #11 from EGA-archive/timeout
Browse files Browse the repository at this point in the history 
new index and making genome ref 38 by default
  • Loading branch information
@costero-e
costero-e authored Nov 12, 2024
2 parents 5d9c2fc + 7480946 commit c32d324
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 21 deletions.
8 changes: 5 additions & 3 deletions beacon/connections/mongo/reindex.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,12 @@
#client.beacon.genomicVariations.create_index([("caseLevelData.biosampleId", 1)])
#client.beacon.genomicVariations.create_index([("variation.location.interval.end.value", -1), ("variation.location.interval.start.value", 1)])
client.beacon.genomicVariations.create_index([("datasetId", 1)])
client.beacon.genomicVariations.create_index([("variantInternalId", 1)])
#client.beacon.genomicVariations.create_index([("variantInternalId", 1)])
client.beacon.genomicVariations.create_index([("variation.location.interval.start.value", 1)])
#client.beacon.genomicVariations.create_index([("variation.location.interval.start.value", 1), ("variation.location.interval.end.value", -1)])
#client.beacon.genomicVariations.create_index([("identifiers.genomicHGVSId", 1), ("variation.location.interval.start.value", 1), ("caseLevelData.biosampleId", 1), ("variation.referenceBases", 1), ("variation.alternateBases", 1)])
client.beacon.genomicVariations.create_index([("variation.location.interval.end.value", -1), ("variation.location.interval.start.value", 1), ("variation.referenceBases", 1), ("variation.alternateBases", 1)])
client.beacon.genomicVariations.create_index([("datasetId", 1), ("variation.location.interval.start.value", 1), ("variation.referenceBases", 1), ("variation.alternateBases", 1)])
#client.beacon.genomicVariations.create_index([("variation.location.interval.end.value", -1), ("variation.location.interval.start.value", 1), ("variation.referenceBases", 1), ("variation.alternateBases", 1)])
#client.beacon.genomicVariations.create_index([("datasetId", 1), ("variation.location.interval.start.value", 1), ("variation.referenceBases", 1), ("variation.alternateBases", 1)])
client.beacon.genomicVariations.create_index([("molecularAttributes.geneIds", 1), ("variantInternalId", 1), ("variation.variantType", 1)])
#client.beacon.individuals.create_index([("$**", "text")])
#client.beacon.runs.create_index([("$**", "text")])
Expand Down
42 changes: 39 additions & 3 deletions beacon/connections/mongo/request_parameters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,36 @@ def generate_position_filter_start(self, key: str, value: List[int]) -> List[Alp
return filters

@log_with_args(level)
def generate_position_filter_end(self, key: str, value: List[int]) -> List[AlphanumericFilter]:
def generate_position_filter_start_equal(self, key: str, value: List[int]) -> List[AlphanumericFilter]:
filters = []
if len(value) == 1:
filters.append(AlphanumericFilter(
id=VARIANTS_PROPERTY_MAP[key],
value=value[0],
operator=Operator.EQUAL
))
elif len(value) == 2:# pragma: no cover
filters.append(AlphanumericFilter(
id=VARIANTS_PROPERTY_MAP[key],
value=value[0],
operator=Operator.GREATER_EQUAL
))
filters.append(AlphanumericFilter(
id=VARIANTS_PROPERTY_MAP[key],
value=value[1],
operator=Operator.LESS_EQUAL
))
return filters

@log_with_args(level)
def generate_position_filter_end(self, key: str, value: List[int]) -> List[AlphanumericFilter]:
filters = []
if len(value) == 1:
filters.append(AlphanumericFilter(
id=VARIANTS_PROPERTY_MAP["start"],
value=value[0],
operator=Operator.LESS
))
elif len(value) == 2:# pragma: no cover
filters.append(AlphanumericFilter(
id=VARIANTS_PROPERTY_MAP[key],
Expand All @@ -79,11 +101,18 @@ def apply_request_parameters(self, query: Dict[str, List[dict]], qparams: Reques
subquery["$and"] = []
subqueryor={}
subqueryor["$or"] = []
equal=True
for k, v in reqparam.items():
if k == 'end':
equal=False
for k, v in reqparam.items():
if k == "start":
if isinstance(v, str):
v = v.split(',')
filters = generate_position_filter_start(self, k, v)
if equal == False:
filters = generate_position_filter_start(self, k, v)
else:
filters = generate_position_filter_start_equal(self, k, v)
for filter in filters:
subquery["$and"].append(apply_alphanumeric_filter({}, filter, collection, dataset))
elif k == "end":
Expand Down Expand Up @@ -149,11 +178,18 @@ def apply_request_parameters(self, query: Dict[str, List[dict]], qparams: Reques
subquery["$and"] = []
subqueryor={}
subqueryor["$or"] = []
equal=True
for k, v in qparams.query.request_parameters.items():
if k == 'end':
equal=False
for k, v in qparams.query.request_parameters.items():
if k == "start":
if isinstance(v, str):
v = v.split(',')
filters = generate_position_filter_start(self, k, v)
if equal == False:
filters = generate_position_filter_start(self, k, v)
else:
filters = generate_position_filter_start_equal(self, k, v)
for filter in filters:
query["$and"].append(apply_alphanumeric_filter(self, {}, filter, collection, dataset))
elif k == "end":
Expand Down
12 changes: 3 additions & 9 deletions beacon/connections/mongo/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,23 +59,16 @@ def get_count(self, collection: Collection, query: dict) -> int:
try:
counts=list(counts)
if counts == []:
match_dict={}
match_dict['$match']=query
count_dict={}
aggregated_query=[]
count_dict["$count"]='Total'
aggregated_query.append(match_dict)
aggregated_query.append(count_dict)
total=list(collection.aggregate(aggregated_query))
total_counts=collection.count_documents(query)
insert_dict={}
insert_dict['id']=str(query)
total_counts=total[0]['Total']
insert_dict['num_results']=total_counts# pragma: no cover
insert_dict['collection']=str(collection)# pragma: no cover
insert_total=client.beacon.counts.insert_one(insert_dict)# pragma: no cover
else:
total_counts=counts[0]["num_results"]
except Exception as e:# pragma: no cover
LOG.debug(e)
insert_dict={}
insert_dict['id']=str(query)
total_counts=0
Expand Down Expand Up @@ -121,6 +114,7 @@ def get_docs_by_response_type(self, include: str, query: dict, dataset: str, lim
queryid={}
queryid['datasetId']=dataset
query_count["$or"].append(queryid)
LOG.debug(query_count)
if query_count["$or"]!=[]:
dataset_count = get_count(self, mongo_collection, query_count)
if dataset_count == 0:
Expand Down
12 changes: 6 additions & 6 deletions beacon/request/parameters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class SequenceQuery(BaseModel):
referenceBases: str
clinicalRelevance: Optional[str] =None
mateName: Optional[str] =None
assemblyId: Optional[str] =None
assemblyId: Optional[str] ='GRCh38'

class RangeQuery(BaseModel):
referenceName: Union[str,int]
Expand All @@ -105,7 +105,7 @@ class RangeQuery(BaseModel):
variantMaxLength: Optional[int] =None
clinicalRelevance: Optional[str] =None
mateName: Optional[str] =None
assemblyId: Optional[str] =None
assemblyId: Optional[str] ='GRCh38'

class DatasetsRequested(BaseModel):
datasets: list
Expand All @@ -117,7 +117,7 @@ class GeneIdQuery(BaseModel):
aminoacidChange: Optional[str] =None
variantMinLength: Optional[int] =None
variantMaxLength: Optional[int] =None
assemblyId: Optional[str] =None
assemblyId: Optional[str] ='GRCh38'

class BracketQuery(BaseModel):
referenceName: Union[str,int]
Expand All @@ -126,7 +126,7 @@ class BracketQuery(BaseModel):
variantType: Optional[str] =None
clinicalRelevance: Optional[str] =None
mateName: Optional[str] =None
assemblyId: Optional[str] =None
assemblyId: Optional[str] ='GRCh38'
@field_validator('start')
@classmethod
def start_must_be_array_of_integers(cls, v: list) -> list:
Expand All @@ -146,12 +146,12 @@ def end_must_be_array_of_integers(cls, v: list) -> list:

class GenomicAlleleQuery(BaseModel):
genomicAlleleShortForm: str
assemblyId: Optional[str] =None
assemblyId: Optional[str] ='GRCh38'

class AminoacidChangeQuery(BaseModel):
aminoacidChange: str
geneId: str
assemblyId: Optional[str] =None
assemblyId: Optional[str] ='GRCh38'

class RequestParams(CamelModel):
meta: RequestMeta = RequestMeta()
Expand Down

0 comments on commit c32d324

Please sign in to comment.