From 610313fae9a98f6e109ab3ab40d7985b802f753f Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:15:22 -0500 Subject: [PATCH 1/3] chore: add telemetry spans --- src/edge_manager.ts | 4 ++++ src/inferred_mode/inferred_mode.ts | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/edge_manager.ts b/src/edge_manager.ts index 2b336565..35b88ea9 100644 --- a/src/edge_manager.ts +++ b/src/edge_manager.ts @@ -397,6 +397,7 @@ export default class QueryEdgeManager { async executeEdges(): Promise { const unavailableAPIs: UnavailableAPITracker = {}; while (this.getEdgesNotExecuted()) { + const span = Telemetry.startSpan({ description: 'edgeExecution' }); //next available/most efficient edge const currentQEdge = this.getNext(); //crate queries from edge @@ -446,6 +447,7 @@ export default class QueryEdgeManager { `qEdge (${currentQEdge.getID()}) got 0 records. Your query terminates.`, ).getLog(), ); + span.finish(); return; } // storing records will trigger a node entity count update @@ -473,6 +475,7 @@ export default class QueryEdgeManager { `qEdge (${currentQEdge.getID()}) kept 0 records. Your query terminates.`, ).getLog(), ); + span.finish(); return; } // edge all done @@ -486,6 +489,7 @@ export default class QueryEdgeManager { if (process.env.DUMP_RECORDS) { await this.dumpRecords(this.getRecords()); } + span.finish(); return true; } } diff --git a/src/inferred_mode/inferred_mode.ts b/src/inferred_mode/inferred_mode.ts index bcf55e22..8f074b6d 100644 --- a/src/inferred_mode/inferred_mode.ts +++ b/src/inferred_mode/inferred_mode.ts @@ -1,5 +1,5 @@ import Debug from 'debug'; -import { LogEntry, StampedLog } from '@biothings-explorer/utils'; +import { LogEntry, StampedLog, Telemetry } from '@biothings-explorer/utils'; import * as utils from '../utils'; import async from 'async'; import biolink from '../biolink'; @@ -254,6 +254,7 @@ export default class InferredQueryHandler { qEdge: TrapiQEdge, combinedResponse: CombinedResponse, ): CombinedResponseReport { + const span = Telemetry.startSpan({ description: 'creativeCombineResponse' }); const newResponse = handler.getResponse(); const report: CombinedResponseReport = { querySuccess: 0, @@ -422,6 +423,7 @@ export default class InferredQueryHandler { if (Object.keys(combinedResponse.message.results).length >= this.CREATIVE_LIMIT && !report.creativeLimitHit) { report.creativeLimitHit = Object.keys(newResponse.message.results).length; } + span.finish(); return report; } @@ -514,8 +516,11 @@ export default class InferredQueryHandler { } = {}; await async.eachOfSeries(subQueries, async ({ template, queryGraph }, i) => { + const span = Telemetry.startSpan({ description: 'creativeTemplate' }); + span.setData('template', i + 1); i = i as number; if (stop) { + span.finish(); return; } if (global.queryInformation?.queryGraph) { @@ -558,6 +563,7 @@ export default class InferredQueryHandler { debug(message); combinedResponse.logs.push(new LogEntry(`INFO`, null, message).getLog()); } + span.finish(); } catch (error) { handler.logs.forEach((log) => { combinedResponse.logs.push(log); @@ -565,6 +571,7 @@ export default class InferredQueryHandler { const message = `ERROR: Template-${i + 1} failed due to error ${error}`; debug(message); combinedResponse.logs.push(new LogEntry(`ERROR`, null, message).getLog()); + span.finish(); return; } }); From 318c1c466c4acc2114f624db9d831bac94bb46e6 Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:15:56 -0500 Subject: [PATCH 2/3] fix: function not async --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 16a0d87a..84e4a47d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -361,7 +361,7 @@ export default class TRAPIQueryHandler { * Set TRAPI Query Graph * @param { object } queryGraph - TRAPI Query Graph Object */ - async setQueryGraph(queryGraph: TrapiQueryGraph): Promise { + setQueryGraph(queryGraph: TrapiQueryGraph): void { this.originalQueryGraph = _.cloneDeep(queryGraph); this.queryGraph = queryGraph; for (const nodeId in queryGraph.nodes) { From c43726114cc0f6fcaf132936110028c9c9688bb5 Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:16:14 -0500 Subject: [PATCH 3/3] perf: mild potential optimizations --- src/edge_manager.ts | 29 +++++++---------------------- src/query_node.ts | 2 +- src/results_assembly/score.ts | 2 +- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/src/edge_manager.ts b/src/edge_manager.ts index 35b88ea9..91da45fe 100644 --- a/src/edge_manager.ts +++ b/src/edge_manager.ts @@ -203,29 +203,14 @@ export default class QueryEdgeManager { const execObjectCuries = qEdge.reverse ? subjectCuries : objectCuries; records.forEach((record) => { - //check sub curies against $input ids - const subjectIDs: Set = new Set(); - const objectIDs: Set = new Set(); - let objectMatch = 0; - let subjectMatch = 0; + // check against original, primaryID, and equivalent ids + const subjectIDs = [record.subject.original, record.subject.curie, ...record.subject.equivalentCuries]; + const objectIDs = [record.object.original, record.object.curie, ...record.object.equivalentCuries]; + + // there must be at least a minimal intersection + const subjectMatch = subjectIDs.some((curie) => execSubjectCuries.includes(curie)); + const objectMatch = objectIDs.some((curie) => execObjectCuries.includes(curie)); - //compare record I/O ids against edge node ids - // #1 check equivalent ids - record.subject.equivalentCuries.forEach((curie) => { - subjectIDs.add(curie); - }); - record.object.equivalentCuries.forEach((curie) => { - objectIDs.add(curie); - }); - // #2 ensure we have the primaryID - subjectIDs.add(record.subject.curie); - objectIDs.add(record.object.curie); - // #3 make sure we at least have the original - subjectIDs.add(record.subject.original); - objectIDs.add(record.object.original); - // check ids - subjectMatch = _.intersection([...subjectIDs], execSubjectCuries).length; - objectMatch = _.intersection([...objectIDs], execObjectCuries).length; //if both ends match then keep record if (subjectMatch && objectMatch) { keep.push(record); diff --git a/src/query_node.ts b/src/query_node.ts index dfacf099..de539a15 100644 --- a/src/query_node.ts +++ b/src/query_node.ts @@ -180,7 +180,7 @@ export default class QNode { const keep: { [mainID: string]: string[] } = {}; // If a new entity has any alias intersection with an existing entity, keep it Object.entries(newCuries).forEach(([newMainID, currentAliases]) => { - const someIntersection = Object.entries(this.expanded_curie).some(([, existingAliases]) => { + const someIntersection = Object.values(this.expanded_curie).some((existingAliases) => { return currentAliases.some((currentAlias) => existingAliases.some((existingAlias) => currentAlias.toLowerCase() === existingAlias.toLowerCase()), ); diff --git a/src/results_assembly/score.ts b/src/results_assembly/score.ts index e28ff983..6cc9e2b4 100644 --- a/src/results_assembly/score.ts +++ b/src/results_assembly/score.ts @@ -24,7 +24,7 @@ export interface ScoreCombos { // create lookup table for ngd scores in the format: {inputUMLS-outputUMLS: ngd} async function query(queryPairs: string[][]): Promise { const url = 'https://biothings.ncats.io/semmeddb/query/ngd'; - const batchSize = 1000; + const batchSize = 500; debug('Querying', queryPairs.length, 'combos.');