diff --git a/__test__/integration/QueryEdge.test.ts b/__test__/integration/QueryEdge.test.ts index b195f01..8a0a18c 100644 --- a/__test__/integration/QueryEdge.test.ts +++ b/__test__/integration/QueryEdge.test.ts @@ -1,5 +1,8 @@ import QNode from '../../src/query_node'; import QEdge from '../../src/query_edge'; +import KGNode from '../../src/graph/kg_node'; +import KGEdge from '../../src/graph/kg_edge'; +import KnowledgeGraph from '../../src/graph/knowledge_graph'; describe('Testing QueryEdge Module', () => { const gene_node1 = new QNode({ id: 'n1', categories: ['Gene'], ids: ['NCBIGene:1017'] }); @@ -197,4 +200,20 @@ describe('Testing QueryEdge Module', () => { // NOTE: recently changed from not.toEqual, because an unfrozen edge *should* equal its original? expect(qEdge1.getHashedEdgeRepresentation()).toEqual(qEdge2.getHashedEdgeRepresentation()); }); + + test('meetsConstraints', () => { + const qEdge = new QEdge({ id: 'e01', subject: type_node, object: disease1_node, predicates: ['biolink:treats'], attribute_constraints: [{ name: 'publications', id: 'biolink:publications', operator: '==', value: 'PMID:9248614', not: false }] }); + const kgNode1 = new KGNode("node1", { label: "node1", semanticType: [], names: [], curies: [], primaryCurie: "node1", qNodeID: "e01"}); + const kgNode2 = new KGNode("node2", { label: "node2", semanticType: [], names: [], curies: [], primaryCurie: "node2", qNodeID: "e01"}); + const kgEdge1 = new KGEdge("edge1", {object: "node1", subject: "node2", predicate: "biolink:treats"}); + const kgEdge2 = new KGEdge("edge2", {object: "node1", subject: "node2", predicate: "biolink:treats"}); + kgEdge1.addPublication("PMID:9248614"); + kgEdge1.addPublication("PMID:1234567"); + kgEdge2.addPublication("PMID:7614243"); + kgEdge2.addPublication("PMID:1234567"); + const graph = new KnowledgeGraph(); + graph.update({ nodes: { node1: kgNode1, node2: kgNode2 }, edges: { edge1: kgEdge1, edge2: kgEdge2 } }); + expect(qEdge.meetsConstraints(graph.edges["edge1"], graph.nodes["node1"], graph.nodes["node2"])).toBeTruthy(); + expect(qEdge.meetsConstraints(graph.edges["edge2"], graph.nodes["node1"], graph.nodes["node2"])).toBeFalsy(); + }) }); diff --git a/__test__/unittest/TRAPIQueryHandler.test.ts b/__test__/unittest/TRAPIQueryHandler.test.ts index 7ea1682..8485d06 100644 --- a/__test__/unittest/TRAPIQueryHandler.test.ts +++ b/__test__/unittest/TRAPIQueryHandler.test.ts @@ -35,7 +35,8 @@ describe('test TRAPIQueryHandler methods', () => { edges: {}, }; try { - await handler._processQueryGraph(invalidQueryGraph); + handler.setQueryGraph(invalidQueryGraph); + await handler._processQueryGraph(); } catch (err) { expect(err).toBeInstanceOf(InvalidQueryGraphError); } diff --git a/src/edge_manager.ts b/src/edge_manager.ts index 79f00dc..57fc3e8 100644 --- a/src/edge_manager.ts +++ b/src/edge_manager.ts @@ -14,6 +14,8 @@ import { SubclassEdges, UnavailableAPITracker } from './types'; import { RecordsByQEdgeID } from './results_assembly/query_results'; import path from 'path'; import { promises as fs } from 'fs'; +import KnowledgeGraph from './graph/knowledge_graph'; +import BTEGraph from './graph/graph'; export default class QueryEdgeManager { private _qEdges: QEdge[]; @@ -250,6 +252,32 @@ export default class QueryEdgeManager { return keep; } + _constrainEdgeRecords(qEdge: QEdge, records: Record[]) { + const keep: Record[] = []; + const bte = new BTEGraph(); + const kg = new KnowledgeGraph(); + bte.update(records); + kg.update(bte); + records.forEach(record => { + const edge = kg.kg.edges[record.recordHash]; + const sub = qEdge.reverse ? kg.kg.nodes[edge.object] : kg.kg.nodes[edge.subject]; + const obj = qEdge.reverse ? kg.kg.nodes[edge.subject] : kg.kg.nodes[edge.object]; + if (qEdge.meetsConstraints(edge, sub, obj)) { + keep.push(record); + } + }); + + debug(`'${qEdge.getID()}' dropped (${records.length - keep.length}) records based on edge/node constraints.`); + this.logs.push( + new LogEntry( + 'DEBUG', + null, + `'${qEdge.getID()}' kept (${keep.length}) / dropped (${records.length - keep.length}) records (based on node/edge constraints).`, + ).getLog(), + ); + return keep; + } + collectRecords(): boolean { //go through edges and collect records organized by edge let recordsByQEdgeID: RecordsByQEdgeID = {}; @@ -316,8 +344,10 @@ export default class QueryEdgeManager { updateEdgeRecords(currentQEdge: QEdge): void { //1. filter edge records based on current status - const filteredRecords = this._filterEdgeRecords(currentQEdge); - //2.trigger node update / entity update based on new status + let filteredRecords = this._filterEdgeRecords(currentQEdge); + //2. make sure node/edge constraints are met + filteredRecords = this._constrainEdgeRecords(currentQEdge, filteredRecords); + //3. trigger node update / entity update based on new status currentQEdge.storeRecords(filteredRecords); } diff --git a/src/graph/knowledge_graph.ts b/src/graph/knowledge_graph.ts index 9aa4b2d..37cec02 100644 --- a/src/graph/knowledge_graph.ts +++ b/src/graph/knowledge_graph.ts @@ -8,6 +8,7 @@ import { TrapiKGNodes, TrapiQualifier, TrapiSource, + TrapiAttributeConstraint, } from '@biothings-explorer/types'; import KGNode from './kg_node'; import KGEdge from './kg_edge'; @@ -199,6 +200,8 @@ export default class KnowledgeGraph { } update(bteGraph: BTEGraphUpdate): void { + this.nodes = {}; + this.edges = {}; Object.keys(bteGraph.nodes).map((node) => { this.nodes[bteGraph.nodes[node].primaryCurie] = this._createNode(bteGraph.nodes[node]); }); diff --git a/src/index.ts b/src/index.ts index 9ce8664..2f2e2d1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -56,6 +56,7 @@ export default class TRAPIQueryHandler { auxGraphs: TrapiAuxGraphCollection; finalizedResults: TrapiResult[]; queryGraph: TrapiQueryGraph; + queryGraphHandler: QueryGraph; constructor( options: QueryHandlerOptions = {}, smartAPIPath: string = undefined, @@ -219,6 +220,9 @@ export default class TRAPIQueryHandler { [edgeID: string]: { [originalSubject: string]: { [originalObject: string]: string /* re-bound edge ID */ } }; } = {}; const edgesIDsByAuxGraphID = {}; + const supportGraphsByEdgeID: { + [edgeID: string]: { [originalSubject: string]: { [originalObject: string]: string /*aux/support graph ID */ } }; + } = {}; Object.entries(this.bteGraph.edges).forEach(([edgeID, bteEdge]) => { if (edgeID.includes('expanded')) return; const combos: { subject: string; object: string; supportGraph: string[] }[] = []; @@ -254,6 +258,10 @@ export default class TRAPIQueryHandler { suffix += 1; } const supportGraphID = `support${suffix}-${boundEdgeID}`; + if (!supportGraphsByEdgeID[edgeID]) supportGraphsByEdgeID[edgeID] = {}; + if (!supportGraphsByEdgeID[edgeID][subject]) supportGraphsByEdgeID[edgeID][subject] = {}; + supportGraphsByEdgeID[edgeID][subject][object] = supportGraphID; + auxGraphs[supportGraphID] = { edges: supportGraph, attributes: [] }; if (!edgesIDsByAuxGraphID[supportGraphID]) { edgesIDsByAuxGraphID[supportGraphID] = new Set(); @@ -305,7 +313,40 @@ export default class TRAPIQueryHandler { newBindings.push(binding); boundIDs.add(binding.id); } - } else if (!boundIDs.has(edgesToRebind[binding.id]?.[subId]?.[objId])) { + } + // we only want to include support graphs that meet constraints (in case there is another QEdge using this same KGEdge) + else if ( + this.queryGraph.edges[qEdgeID].attribute_constraints || + this.queryGraph.nodes[this.queryGraph.edges[qEdgeID].subject].constraints || + this.queryGraph.nodes[this.queryGraph.edges[qEdgeID].object].constraints + ) { + const oldBoundEdge = this.bteGraph.edges[edgesToRebind[binding.id]?.[subId]?.[objId]]; + const newBoundEdge = `${edgesToRebind[binding.id][subId][objId]}-constrained_by_${qEdgeID}`; + if (!boundIDs.has(newBoundEdge)) { + this.bteGraph.edges[newBoundEdge] = new KGEdge(newBoundEdge, { + predicate: oldBoundEdge.predicate, + subject: oldBoundEdge.subject, + object: oldBoundEdge.object, + }); + this.bteGraph.edges[newBoundEdge].addAdditionalAttributes('biolink:support_graphs', []); + this.bteGraph.edges[newBoundEdge].addAdditionalAttributes('biolink:knowledge_level', 'logical_entailment') + this.bteGraph.edges[newBoundEdge].addAdditionalAttributes('biolink:agent_type', 'automated_agent') + this.bteGraph.edges[newBoundEdge].addSource([ + { + resource_id: this.options.provenanceUsesServiceProvider + ? 'infores:service-provider-trapi' + : 'infores:biothings-explorer', + resource_role: 'primary_knowledge_source', + }, + ]); + boundIDs.add(newBoundEdge); + newBindings.push({...binding, id: newBoundEdge }); + resultBoundEdgesWithAuxGraphs.add(newBoundEdge); + } + (this.bteGraph.edges[newBoundEdge].attributes['biolink:support_graphs'] as Set).add(supportGraphsByEdgeID[binding.id][subId][objId]); + edgesIDsByAuxGraphID[supportGraphsByEdgeID[binding.id][subId][objId]].add(newBoundEdge); + } + else if (!boundIDs.has(edgesToRebind[binding.id]?.[subId]?.[objId])) { newBindings.push({ id: edgesToRebind[binding.id]?.[subId]?.[objId], attributes: [] }); boundIDs.add(edgesToRebind[binding.id]?.[subId]?.[objId]); resultBoundEdgesWithAuxGraphs.add(edgesToRebind[binding.id]?.[subId]?.[objId]); @@ -468,6 +509,7 @@ export default class TRAPIQueryHandler { } } } + this.queryGraphHandler = new QueryGraph(queryGraph, this.options.schema, this._queryIsPathfinder()); } _initializeResponse(): void { @@ -477,11 +519,19 @@ export default class TRAPIQueryHandler { this.bteGraph.subscribe(this.knowledgeGraph); } - async _processQueryGraph(queryGraph: TrapiQueryGraph): Promise { - const queryGraphHandler = new QueryGraph(queryGraph, this.options.schema, this._queryIsPathfinder()); - const queryEdges = await queryGraphHandler.calculateEdges(); - this.logs = [...this.logs, ...queryGraphHandler.logs]; - return queryEdges; + async _processQueryGraph(): Promise { + try { + const queryEdges = await this.queryGraphHandler.calculateEdges(); + this.logs = [...this.logs, ...this.queryGraphHandler.logs]; + return queryEdges; + } catch (err) { + if (err instanceof InvalidQueryGraphError || err instanceof SRINodeNormFailure) { + throw err; + } else { + console.log(err.stack); + throw new InvalidQueryGraphError(); + } + } } async _edgesSupported(qEdges: QEdge[], metaKG: MetaKG): Promise { @@ -608,6 +658,9 @@ export default class TRAPIQueryHandler { this.logs.push(new LogEntry('WARNING', null, message).getLog()); return; } + if (await this._checkContraints()) { + return; + } const inferredQueryHandler = new InferredQueryHandler( this, this.queryGraph, @@ -644,7 +697,7 @@ export default class TRAPIQueryHandler { new LogEntry( 'ERROR', null, - `BTE does not currently support any type of constraint. Your query Terminates.`, + `BTE does not currently support constraints with creative mode. Your query Terminates.`, ).getLog(), ); return true; @@ -737,11 +790,8 @@ export default class TRAPIQueryHandler { ); } - const queryEdges = await this._processQueryGraph(this.queryGraph); - // TODO remove this when constraints implemented - if (await this._checkContraints()) { - return; - } + const queryEdges = await this._processQueryGraph(); + if ((this.options.smartAPIID || this.options.teamName) && Object.values(this.queryGraph.edges).length > 1) { const message = 'smartAPI/team-specific endpoints only support single-edge queries. Your query terminates.'; this.logs.push(new LogEntry('WARNING', null, message).getLog()); @@ -779,10 +829,11 @@ export default class TRAPIQueryHandler { // update query graph this.bteGraph.update(manager.getRecords()); + this.bteGraph.notify(); //update query results await this.trapiResultsAssembler.update( manager.getOrganizedRecords(), - !(this.options.smartAPIID || this.options.teamName), + !(this.options.smartAPIID || this.options.teamName) ); this.logs = [...this.logs, ...this.trapiResultsAssembler.logs]; // fix subclassing diff --git a/src/query_edge.ts b/src/query_edge.ts index 2905511..1e75e43 100644 --- a/src/query_edge.ts +++ b/src/query_edge.ts @@ -5,7 +5,7 @@ import QNode from './query_node'; import { QNodeInfo } from './query_node'; import * as utils from '@biothings-explorer/utils'; import { LogEntry, StampedLog } from '@biothings-explorer/utils'; -import { TrapiAttributeConstraint, TrapiQualifierConstraint } from '@biothings-explorer/types'; +import { TrapiAttribute, TrapiAttributeConstraint, TrapiKGEdge, TrapiKGNode, TrapiQualifierConstraint } from '@biothings-explorer/types'; const debug = Debug('bte:biothings-explorer-trapi:QEdge'); @@ -31,6 +31,7 @@ interface QEdgeInfo { executed?: boolean; reverse?: boolean; qualifier_constraints?: TrapiQualifierConstraint[]; + attribute_constraints?: TrapiAttributeConstraint[]; frozen?: boolean; predicates?: string[]; } @@ -50,6 +51,7 @@ export default class QEdge { object: QNode; expanded_predicates: string[]; qualifier_constraints: TrapiQualifierConstraint[]; + constraints: TrapiAttributeConstraint[]; reverse: boolean; executed: boolean; logs: StampedLog[]; @@ -63,6 +65,7 @@ export default class QEdge { this.object = info.frozen === true ? new QNode(info.object as QNodeInfo) : (info.object as QNode); this.expanded_predicates = []; this.qualifier_constraints = info.qualifier_constraints || []; + this.constraints = info.attribute_constraints || []; this.reverse = this.subject?.getCurie?.() === undefined && this.object?.getCurie?.() !== undefined; @@ -350,188 +353,12 @@ export default class QEdge { !this.reverse ? this.subject.updateCuries(combined_curies_2) : this.object.updateCuries(combined_curies_2); } - applyNodeConstraints(): void { - debug(`(6) Applying Node Constraints to ${this.records.length} records.`); - const kept = []; - let save_kept = false; - const sub_constraints = this.subject.constraints; - if (sub_constraints && sub_constraints.length) { - const from = this.reverse ? 'object' : 'subject'; - debug(`Node (subject) constraints: ${JSON.stringify(sub_constraints)}`); - save_kept = true; - for (let i = 0; i < this.records.length; i++) { - const res = this.records[i]; - let keep = true; - // apply constraints - for (let x = 0; x < sub_constraints.length; x++) { - const constraint = sub_constraints[x]; - keep = this.meetsConstraint(constraint, res, from); - } - // pass or not - if (keep) { - kept.push(res); - } - } - } - - const obj_constraints = this.object.constraints; - if (obj_constraints && obj_constraints.length) { - const from = this.reverse ? 'subject' : 'object'; - debug(`Node (object) constraints: ${JSON.stringify(obj_constraints)}`); - save_kept = true; - for (let i = 0; i < this.records.length; i++) { - const res = this.records[i]; - let keep = true; - // apply constraints - for (let x = 0; x < obj_constraints.length; x++) { - const constraint = obj_constraints[x]; - keep = this.meetsConstraint(constraint, res, from); - } - // pass or not - if (keep) { - kept.push(res); - } - } - } - if (save_kept) { - // only override recordss if there was any filtering done. - this.records = kept; - debug(`(6) Reduced to (${this.records.length}) records.`); - } else { - debug(`(6) No constraints. Skipping...`); - } - } - - meetsConstraint(constraint: TrapiAttributeConstraint, record: Record, from: string): boolean { - // list of attribute ids in node - const available_attributes = [...new Set(Object.keys(record[from].attributes))]; - // debug(`ATTRS ${JSON.stringify(record[from].normalizedInfo[0]._leafSemanticType)}` + - // ` ${from} : ${JSON.stringify(available_attributes)}`); - // determine if node even contains right attributes - const filters_found = available_attributes.filter((attr) => attr == constraint.id); - if (!filters_found.length) { - // node doesn't have the attribute needed - return false; - } else { - // match attr by name, parse only attrs of interest - const node_attributes = {}; - filters_found.forEach((filter) => { - node_attributes[filter] = record[from].attributes[filter]; - }); - switch (constraint.operator) { - case '==': - for (const key in node_attributes) { - if (!isNaN(constraint.value as number)) { - if (Array.isArray(node_attributes[key])) { - if ( - node_attributes[key].includes(constraint.value) || - node_attributes[key].includes(constraint.value.toString()) - ) { - return true; - } - } else { - if ( - node_attributes[key] == constraint.value || - node_attributes[key] == constraint.value.toString() || - node_attributes[key] == parseInt(constraint.value as string) - ) { - return true; - } - } - } else { - if (Array.isArray(node_attributes[key])) { - if (node_attributes[key].includes(constraint.value)) { - return true; - } - } else { - if ( - node_attributes[key] == constraint.value || - node_attributes[key] == constraint.value.toString() || - node_attributes[key] == parseInt(constraint.value as string) - ) { - return true; - } - } - } - } - return false; - case '>': - for (const key in node_attributes) { - if (Array.isArray(node_attributes[key])) { - for (let index = 0; index < node_attributes[key].length; index++) { - const element = node_attributes[key][index]; - if (parseInt(element) > parseInt(constraint.value as string)) { - return true; - } - } - } else { - if (parseInt(node_attributes[key]) > parseInt(constraint.value as string)) { - return true; - } - } - } - return false; - case '>=': - for (const key in node_attributes) { - if (Array.isArray(node_attributes[key])) { - for (let index = 0; index < node_attributes[key].length; index++) { - const element = node_attributes[key][index]; - if (parseInt(element) >= parseInt(constraint.value as string)) { - return true; - } - } - } else { - if (parseInt(node_attributes[key]) >= parseInt(constraint.value as string)) { - return true; - } - } - } - return false; - case '<': - for (const key in node_attributes) { - if (Array.isArray(node_attributes[key])) { - for (let index = 0; index < node_attributes[key].length; index++) { - const element = node_attributes[key][index]; - if (parseInt(element) > parseInt(constraint.value as string)) { - return true; - } - } - } else { - if (parseInt(node_attributes[key]) < parseInt(constraint.value as string)) { - return true; - } - } - } - return false; - case '<=': - for (const key in node_attributes) { - if (Array.isArray(node_attributes[key])) { - for (let index = 0; index < node_attributes[key].length; index++) { - const element = node_attributes[key][index]; - if (parseInt(element) <= parseInt(constraint.value as string)) { - return true; - } - } - } else { - if (parseInt(node_attributes[key]) <= parseInt(constraint.value as string)) { - return true; - } - } - } - return false; - default: - debug(`Node operator not handled ${constraint.operator}`); - return false; - } - } - } - + storeRecords(records: Record[]): void { debug(`(6) Storing records...`); // store new records in current edge this.records = records; // will update records if any constraints are found - this.applyNodeConstraints(); debug(`(7) Updating nodes based on edge records...`); this.updateNodesCuries(records); } @@ -576,4 +403,83 @@ export default class QEdge { getReversedPredicate(predicate: string): string { return predicate ? utils.biolink.reverse(predicate) : undefined; } + + meetsConstraints(kgEdge: TrapiKGEdge, kgSub: TrapiKGNode, kgObj: TrapiKGNode): boolean { + // edge constraints + if (this.constraints) { + for (let constraint of this.constraints) { + let meets = this._meetsConstraint(constraint, kgEdge.attributes); + if (constraint.not) meets = !meets; + if (!meets) return false; + } + } + + // node constraints not fully tested yet (may be some weird behavior with subclsasing) + // subject constraints + if (this.subject.constraints) { + for (let constraint of this.subject.constraints) { + let meets = this._meetsConstraint(constraint, kgSub.attributes); + if (constraint.not) meets = !meets; + if (!meets) return false; + } + } + + // object constraints + if (this.object.constraints) { + for (let constraint of this.object.constraints) { + let meets = this._meetsConstraint(constraint, kgObj.attributes); + if (constraint.not) meets = !meets; + if (!meets) return false; + } + } + + return true; + } + + _meetsConstraint(constraint: TrapiAttributeConstraint, attributes?: TrapiAttribute[]): boolean { + const edge_attribute = attributes?.find(x => x.attribute_type_id == constraint.id)?.value as any; + const constraintValue = constraint.value as any; + if (!edge_attribute) { + return false; + } + switch (constraint.operator) { + case '==': + const array1 = utils.toArray(edge_attribute); + const array2 = utils.toArray(constraintValue); + for (let a1 of array1) { + for (let a2 of array2) { + if (a1 == a2) return true; + } + } + return false; + case '===': + if (Array.isArray(edge_attribute) && Array.isArray(constraintValue)) { + if (edge_attribute.length !== constraintValue.length) return false; + for (let i = 0; i < edge_attribute.length; i++) { + if (edge_attribute[i] !== constraintValue[i]) return false; + } + return true; + } + return edge_attribute === constraintValue; + case 'matches': + if (typeof constraintValue === 'string') { + let regexStr = constraintValue; + // make sure regex matches the whole string + if (constraintValue.at(0) !== '^') regexStr = '^' + regexStr; + if (constraintValue.at(constraintValue.length - 1) !== '$') regexStr += '$'; + let regex = new RegExp(regexStr); + for (let attr of utils.toArray(edge_attribute)) { + if (regex.test(attr)) return true; + } + } + return false; + case '>': + return edge_attribute > constraintValue; + case '<': + return edge_attribute < constraintValue; + default: + debug(`Node operator not handled ${constraint.operator}`); + return false; + } + } } diff --git a/src/results_assembly/query_results.ts b/src/results_assembly/query_results.ts index 9c44491..39d6814 100644 --- a/src/results_assembly/query_results.ts +++ b/src/results_assembly/query_results.ts @@ -1,11 +1,12 @@ import { LogEntry, StampedLog } from '@biothings-explorer/utils'; -import { TrapiResult } from '@biothings-explorer/types'; +import { TrapiKnowledgeGraph, TrapiResult } from '@biothings-explorer/types'; import Debug from 'debug'; import { zip } from 'lodash'; const debug = Debug('bte:biothings-explorer-trapi:QueryResult'); import { getScores, calculateScore, ScoreCombos } from './score'; import { Record } from '@biothings-explorer/api-response-transform'; import * as config from '../config'; +import QueryGraph from '../query_graph'; export interface RecordsByQEdgeID { [qEdgeID: string]: {