Merge branch 'pathfinder' of https://github.com/biothings/bte_trapi_q…

…uery_graph_handler into pathfinder-simpleCaseB
biothings · Apr 18, 2024 · d74eca4 · d74eca4
2 parents 4c8598e + 78c0c84
commit d74eca4
Show file tree

Hide file tree

Showing 15 changed files with 84 additions and 201 deletions.
diff --git a/src/cache_handler.ts b/src/cache_handler.ts
@@ -9,7 +9,7 @@ import chunker from 'stream-chunker';
 import { Readable, Transform } from 'stream';
 import { Record, RecordPackage } from '@biothings-explorer/api-response-transform';
 import { threadId } from 'worker_threads';
-import MetaKG from '../../smartapi-kg/built';
+import MetaKG from '@biothings-explorer/smartapi-kg';
 import QEdge from './query_edge';
 import { QueryHandlerOptions } from '@biothings-explorer/types';
 
@@ -113,7 +113,7 @@ export default class CacheHandler {
   }
 
   async categorizeEdges(qEdges: QEdge[]): Promise<{ cachedRecords: Record[]; nonCachedQEdges: QEdge[] }> {
-    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === "true") {
+    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === 'true') {
       return {
         cachedRecords: [],
         nonCachedQEdges: qEdges,
@@ -210,7 +210,7 @@ export default class CacheHandler {
   }
 
   async cacheEdges(queryRecords: Record[]): Promise<void> {
-    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === "true") {
+    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === 'true') {
       if (global.parentPort) {
         global.parentPort.postMessage({ threadId, cacheDone: true });
       }
@@ -257,8 +257,8 @@ export default class CacheHandler {
                   resolve();
                 });
             });
-            if (process.env.QEDGE_CACHE_TIME_S !== "0") {
-                await redisClient.client.expireTimeout(redisID, process.env.QEDGE_CACHE_TIME_S || 1800);
+            if (process.env.QEDGE_CACHE_TIME_S !== '0') {
+              await redisClient.client.expireTimeout(redisID, process.env.QEDGE_CACHE_TIME_S || 1800);
             }
           } catch (error) {
             failedHashes.push(hash);

diff --git a/src/graph/graph.ts b/src/graph/graph.ts
@@ -4,8 +4,8 @@ import Debug from 'debug';
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
 import KGNode from './kg_node';
 import KGEdge from './kg_edge';
-import { Record } from '../../../api-response-transform/built';
-import { TrapiAuxiliaryGraph, TrapiResult } from '../types';
+import { Record } from '@biothings-explorer/api-response-transform';
+import { TrapiAuxiliaryGraph, TrapiResult } from '@biothings-explorer/types';
 import KnowledgeGraph from './knowledge_graph';
 const debug = Debug('bte:biothings-explorer-trapi:Graph');
 
@@ -104,6 +104,12 @@ export default class BTEGraph {
           .map((item) => {
             this.edges[recordHash].addAdditionalAttributes(item, record.mappedResponse[item]);
           });
+        if (record.knowledge_level) {
+          this.edges[recordHash].addAdditionalAttributes('biolink:knowledge_level', record.knowledge_level);
+        }
+        if (record.agent_type) {
+          this.edges[recordHash].addAdditionalAttributes('biolink:agent_type', record.agent_type);
+        }
         this.edges[recordHash].addSource(record.provenanceChain);
         Object.entries(record.qualifiers).forEach(([qualifierType, qualifier]) => {
           this.edges[recordHash].addQualifier(qualifierType, qualifier);

diff --git a/src/graph/kg_edge.ts b/src/graph/kg_edge.ts
@@ -1,5 +1,5 @@
-import { ProvenanceChainItem } from '@biothings-explorer/api-response-transform';
-import { TrapiAttribute } from '../types';
+import { TrapiSource } from '@biothings-explorer/types';
+import { TrapiAttribute } from '@biothings-explorer/types';
 
 export interface KGEdgeInfo {
   object: string;
@@ -20,6 +20,7 @@ export default class KGEdge {
         resource_id: string;
         resource_role: string;
         upstream_resource_ids?: Set<string>;
+        source_record_urls?: Set<string>;
       };
     };
   };
@@ -68,7 +69,7 @@ export default class KGEdge {
     });
   }
 
-  addSource(source: ProvenanceChainItem | ProvenanceChainItem[]): void {
+  addSource(source: TrapiSource | TrapiSource[]): void {
     if (typeof source === 'undefined') {
       return;
     }
@@ -77,19 +78,26 @@ export default class KGEdge {
     }
     source.forEach((item) => {
       if (!this.sources[item.resource_id]) this.sources[item.resource_id] = {};
+      if (item.upstream_resource_ids && !Array.isArray(item.upstream_resource_ids)) {
+        item.upstream_resource_ids = [item.upstream_resource_ids];
+      }
+      if (item.source_record_urls && !Array.isArray(item.source_record_urls)) {
+        item.source_record_urls = [item.source_record_urls];
+      }
       if (!this.sources[item.resource_id][item.resource_role]) {
         this.sources[item.resource_id][item.resource_role] = {
           resource_id: item.resource_id,
           resource_role: item.resource_role,
           upstream_resource_ids: item.upstream_resource_ids ? new Set(item.upstream_resource_ids) : undefined,
+          source_record_urls: item.source_record_urls ? new Set(item.source_record_urls) : undefined,
         };
       }
-      if (item.upstream_resource_ids && !Array.isArray(item.upstream_resource_ids)) {
-        item.upstream_resource_ids = [item.upstream_resource_ids];
-      }
       item.upstream_resource_ids?.forEach((upstream) =>
         this.sources[item.resource_id][item.resource_role].upstream_resource_ids.add(upstream),
       );
+      item.source_record_urls?.forEach((url) =>
+        this.sources[item.resource_id][item.resource_role].source_record_urls.add(url),
+      );
     });
   }
 

diff --git a/src/graph/kg_node.ts b/src/graph/kg_node.ts
@@ -1,4 +1,4 @@
-import { TrapiAttribute } from '../types';
+import { TrapiAttribute } from '@biothings-explorer/types';
 
 export interface KGNodeInfo {
   label: string;

diff --git a/src/graph/knowledge_graph.ts b/src/graph/knowledge_graph.ts
@@ -9,14 +9,16 @@ import {
   TrapiKGNodes,
   TrapiQualifier,
   TrapiSource,
-} from '../types';
+} from '@biothings-explorer/types';
 import KGNode from './kg_node';
 import KGEdge from './kg_edge';
 import { BTEGraphUpdate } from './graph';
 import { APIDefinition } from '@biothings-explorer/types';
 
 const debug = Debug('bte:biothings-explorer-trapi:KnowledgeGraph');
 
+const NON_ARRAY_ATTRIBUTES = ['biolink:knowledge_level', 'biolink:agent_type'];
+
 export default class KnowledgeGraph {
   nodes: {
     [nodePrimaryID: string]: TrapiKGNode;
@@ -114,11 +116,14 @@ export default class KnowledgeGraph {
     }
 
     Object.entries(kgEdge.attributes).forEach(([key, value]) => {
-      if (key == 'edge-attributes') return;
+      if (key === 'edge-attributes') return;
       // if (key == 'edge-attributes') return;
       attributes.push({
         attribute_type_id: key,
-        value: Array.from(value as Set<string>),
+        value: // technically works for numbers as well
+          NON_ARRAY_ATTRIBUTES.includes(key)
+            ? [...(value as Set<string>)].reduce((acc, val) => acc + val)
+            : Array.from(value as Set<string>),
         //value_type_id: 'bts:' + key,
       });
     });
@@ -137,6 +142,7 @@ export default class KnowledgeGraph {
         const trapiSource: TrapiSource = {
           ...sourceObj,
           upstream_resource_ids: sourceObj.upstream_resource_ids ? [...sourceObj.upstream_resource_ids] : undefined,
+          source_record_urls: sourceObj.source_record_urls ? [...sourceObj.source_record_urls] : undefined,
         };
         sources.push(trapiSource);
       });

diff --git a/src/index.ts b/src/index.ts
@@ -25,7 +25,7 @@ import {
   TrapiQueryGraph,
   TrapiResponse,
   TrapiResult,
-} from './types';
+} from '@biothings-explorer/types';
 import { QueryHandlerOptions } from '@biothings-explorer/types';
 import BTEGraph from './graph/graph';
 import QEdge from './query_edge';
@@ -174,6 +174,8 @@ export default class TRAPIQueryHandler {
         const source = Object.entries(ontologyKnowledgeSourceMapping).find(([prefix]) => {
           return expanded.includes(prefix);
         })[1];
+        subclassEdge.addAdditionalAttributes('biolink:knowledge_level', 'knowledge_assertion')
+        subclassEdge.addAdditionalAttributes('biolink:agent_type', 'manual_agent')
         subclassEdge.addSource([
           { resource_id: source, resource_role: 'primary_knowledge_source' },
           {
@@ -210,7 +212,7 @@ export default class TRAPIQueryHandler {
         suffix += 1;
       }
       const supportGraphID = `support${suffix}-${boundEdgeID}`;
-      auxGraphs[supportGraphID] = { edges: supportGraph };
+      auxGraphs[supportGraphID] = { edges: supportGraph, attributes: [] };
       if (!edgesIDsByAuxGraphID[supportGraphID]) {
         edgesIDsByAuxGraphID[supportGraphID] = new Set();
       }
@@ -222,6 +224,8 @@ export default class TRAPIQueryHandler {
           object: object,
         });
         boundEdge.addAdditionalAttributes('biolink:support_graphs', [supportGraphID]);
+        boundEdge.addAdditionalAttributes('biolink:knowledge_level', 'logical_entailment')
+        boundEdge.addAdditionalAttributes('biolink:agent_type', 'automated_agent')
         boundEdge.addSource([
           {
             resource_id: this.options.provenanceUsesServiceProvider
@@ -251,7 +255,7 @@ export default class TRAPIQueryHandler {
                     boundIDs.add(binding.id);
                   }
                 } else if (!boundIDs.has(nodesToRebind[binding.id].newNode)) {
-                  newBindings.push({ id: nodesToRebind[binding.id].newNode });
+                  newBindings.push({ id: nodesToRebind[binding.id].newNode, attributes: [] });
                   boundIDs.add(nodesToRebind[binding.id].newNode);
                 }
                 return { boundIDs, newBindings };
@@ -273,7 +277,7 @@ export default class TRAPIQueryHandler {
                     boundIDs.add(binding.id);
                   }
                 } else if (!boundIDs.has(edgesToRebind[binding.id])) {
-                  newBindings.push({ id: edgesToRebind[binding.id] });
+                  newBindings.push({ id: edgesToRebind[binding.id], attributes: [] });
                   boundIDs.add(edgesToRebind[binding.id]);
                   resultBoundEdgesWithAuxGraphs.add(edgesToRebind[binding.id]);
                 }
@@ -334,7 +338,7 @@ export default class TRAPIQueryHandler {
       description: `Query processed successfully, retrieved ${results.length} results.`,
       schema_version: global.SCHEMA_VERSION,
       biolink_version: global.BIOLINK_VERSION,
-      workflow: [{ id: 'lookup' }],
+      workflow: [{ id: this.options.smartAPIID || this.options.teamName ? 'lookup' : 'lookup_and_score' }],
       message: {
         query_graph: this.originalQueryGraph,
         knowledge_graph: this.knowledgeGraph.kg,

diff --git a/src/inferred_mode/inferred_mode.ts b/src/inferred_mode/inferred_mode.ts
@@ -6,9 +6,8 @@ import biolink from '../biolink';
 import { getTemplates, MatchedTemplate, TemplateLookup } from './template_lookup';
 import { scaled_sigmoid, inverse_scaled_sigmoid } from '../results_assembly/score';
 import TRAPIQueryHandler from '../index';
-import { QueryHandlerOptions } from '@biothings-explorer/types';
 import {
-  CompactQualifiers,
+  QueryHandlerOptions,
   TrapiAuxGraphCollection,
   TrapiEdgeBinding,
   TrapiKnowledgeGraph,
@@ -18,7 +17,8 @@ import {
   TrapiResponse,
   TrapiResult,
   TrapiAnalysis
-} from '../types';
+} from '@biothings-explorer/types';
+import { CompactQualifiers } from '../index';
 const debug = Debug('bte:biothings-explorer-trapi:inferred-mode');
 
 export interface CombinedResponse {
@@ -293,12 +293,13 @@ export default class InferredQueryHandler {
     // modified count used for pathfinder
     const pfIntermediateSet = new Set();
 
+    let auxGraphSuffixes: {[inferredEdgeID: string]: number} = {};
     // add results
     newResponse.message.results.forEach((result) => {
       const translatedResult: TrapiResult = {
         node_bindings: {
-          [qEdge.subject]: [{ id: result.node_bindings.creativeQuerySubject[0].id }],
-          [qEdge.object]: [{ id: result.node_bindings.creativeQueryObject[0].id }],
+          [qEdge.subject]: [{ id: result.node_bindings.creativeQuerySubject[0].id, attributes: [] }],
+          [qEdge.object]: [{ id: result.node_bindings.creativeQueryObject[0].id, attributes: [] }],
         },
         pfocr: result.pfocr?.length ? result.pfocr : undefined,
         analyses: [
@@ -332,14 +333,14 @@ export default class InferredQueryHandler {
       // Direct edge answers stand on their own, not as an inferred edge.
       if (Object.keys(result.node_bindings).length == 2) {
         const boundEdgeID = Object.values(result.analyses[0].edge_bindings)[0][0].id;
-        translatedResult.analyses[0].edge_bindings = { [qEdgeID]: [{ id: boundEdgeID }] };
+        translatedResult.analyses[0].edge_bindings = { [qEdgeID]: [{ id: boundEdgeID, attributes: [] }] };
       } else {
         // Create an aux graph using the result and associate it with an inferred Edge
         const inferredEdgeID = `inferred-${resultCreativeSubjectID}-${qEdge.predicates[0].replace(
           'biolink:',
           '',
         )}-${resultCreativeObjectID}`;
-        translatedResult.analyses[0].edge_bindings = { [qEdgeID]: [{ id: inferredEdgeID }] };
+        translatedResult.analyses[0].edge_bindings = { [qEdgeID]: [{ id: inferredEdgeID, attributes: [] }] };
         if (!combinedResponse.message.knowledge_graph.edges[inferredEdgeID]) {
           combinedResponse.message.knowledge_graph.edges[inferredEdgeID] = {
             subject: resultCreativeSubjectID,
@@ -353,16 +354,16 @@ export default class InferredQueryHandler {
                 resource_role: 'primary_knowledge_source',
               },
             ],
-            attributes: [{ attribute_type_id: 'biolink:support_graphs', value: [] }],
+            attributes: [
+              { attribute_type_id: 'biolink:support_graphs', value: [] },
+              { attribute_type_id: 'biolink:knowledge_level', value: "prediction" },
+              { attribute_type_id: 'biolink:agent_type', value: "computational_model" },
+            ],
           };
         }
-        let auxGraphSuffix = 0;
-        while (
-          Object.keys(combinedResponse.message.auxiliary_graphs).includes(`${inferredEdgeID}-support${auxGraphSuffix}`)
-        ) {
-          auxGraphSuffix += 1;
-        }
-        const auxGraphID = `${inferredEdgeID}-support${auxGraphSuffix}`;
+        if (!auxGraphSuffixes[inferredEdgeID]) auxGraphSuffixes[inferredEdgeID] = 0;
+        const auxGraphID = `${inferredEdgeID}-support${auxGraphSuffixes[inferredEdgeID]}`;
+        auxGraphSuffixes[inferredEdgeID]++;
         (combinedResponse.message.knowledge_graph.edges[inferredEdgeID].attributes[0].value as string[]).push(
           auxGraphID,
         );
@@ -374,6 +375,7 @@ export default class InferredQueryHandler {
             },
             [] as string[],
           ),
+          attributes: []
         };
 
         if (this.pathfinder) {
@@ -410,9 +412,9 @@ export default class InferredQueryHandler {
         if (typeof combinedResponse.message.results[resultID].analyses[0].score !== 'undefined') {
           combinedResponse.message.results[resultID].analyses[0].score = resScore
             ? scaled_sigmoid(
-              inverse_scaled_sigmoid(combinedResponse.message.results[resultID].analyses[0].score) +
-              inverse_scaled_sigmoid(resScore),
-            )
+                inverse_scaled_sigmoid(combinedResponse.message.results[resultID].analyses[0].score) +
+                  inverse_scaled_sigmoid(resScore),
+              )
             : combinedResponse.message.results[resultID].analyses[0].score;
         } else {
           combinedResponse.message.results[resultID].analyses[0].score = resScore;
@@ -524,7 +526,7 @@ export default class InferredQueryHandler {
       description: '',
       schema_version: global.SCHEMA_VERSION,
       biolink_version: global.BIOLINK_VERSION,
-      workflow: [{ id: 'lookup' }],
+      workflow: [{ id: 'lookup_and_score' }],
       message: {
         query_graph: this.queryGraph,
         knowledge_graph: {
@@ -585,7 +587,8 @@ export default class InferredQueryHandler {
             `creative result maximum of ${this.CREATIVE_LIMIT} (reaching ${
               creativeLimitHit
             } merged). `,
-            `Response will be truncated to top-scoring ${this.CREATIVE_LIMIT} results. Skipping remaining ${subQueries.length - (i + 1)
+            `Response will be truncated to top-scoring ${this.CREATIVE_LIMIT} results. Skipping remaining ${
+              subQueries.length - (i + 1)
             } `,
             subQueries.length - (i + 1) === 1 ? `template.` : `templates.`,
           ].join('');
@@ -610,8 +613,9 @@ export default class InferredQueryHandler {
       const total =
         Object.values(mergedResultsCount).reduce((sum, count) => sum + count, 0) +
         Object.keys(mergedResultsCount).length;
-      const message = `Merging Summary: (${total}) inferred-template results were merged into (${Object.keys(mergedResultsCount).length
-        }) final results, reducing result count by (${total - Object.keys(mergedResultsCount).length})`;
+      const message = `Merging Summary: (${total}) inferred-template results were merged into (${
+        Object.keys(mergedResultsCount).length
+      }) final results, reducing result count by (${total - Object.keys(mergedResultsCount).length})`;
       debug(message);
       combinedResponse.logs.push(new LogEntry('INFO', null, message).getLog());
     }

diff --git a/src/inferred_mode/pathfinder.ts b/src/inferred_mode/pathfinder.ts
@@ -1,9 +1,8 @@
 import TRAPIQueryHandler from '../index';
-import { TrapiResponse, TrapiQEdge, TrapiResult, TrapiQueryGraph, TrapiQNode, TrapiAnalysis } from '../types';
+import { TrapiResponse, TrapiQEdge, TrapiResult, TrapiQueryGraph, TrapiQNode, TrapiAnalysis, QueryHandlerOptions } from '@biothings-explorer/types';
 import InferredQueryHandler from './inferred_mode';
 import { scaled_sigmoid, inverse_scaled_sigmoid } from '../results_assembly/score';
 import { LogEntry, StampedLog, Telemetry } from '@biothings-explorer/utils';
-import { QueryHandlerOptions } from '@biothings-explorer/types';
 import Debug from 'debug';
 const debug = Debug('bte:biothings-explorer-trapi:pathfinder');
 
@@ -313,4 +312,4 @@ export default class PathfinderQueryHandler {
 
     return { results: newResultObject, graphs: newAuxGraphs };
   }
-}
+}
diff --git a/src/inferred_mode/template_lookup.ts b/src/inferred_mode/template_lookup.ts
@@ -2,7 +2,8 @@
 import { promises as fs } from 'fs';
 import path from 'path';
 import async from 'async';
-import { TrapiQueryGraph, CompactQualifiers } from '../types';
+import { TrapiQueryGraph } from '@biothings-explorer/types';
+import { CompactQualifiers } from '../types';
 
 export interface TemplateLookup {
   subject: string;

diff --git a/src/query_edge.ts b/src/query_edge.ts
@@ -6,7 +6,7 @@ import { Record, RecordNode, FrozenRecord } from '@biothings-explorer/api-respon
 import QNode from './query_node';
 import { QNodeInfo } from './query_node';
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
-import { TrapiAttributeConstraint, TrapiQualifierConstraint } from './types';
+import { TrapiAttributeConstraint, TrapiQualifierConstraint } from '@biothings-explorer/types';
 
 const debug = Debug('bte:biothings-explorer-trapi:QEdge');