Merge branch 'main' of https://github.com/biothings/bte_trapi_query_g…

…raph_handler into dev
biothings · Apr 12, 2024 · 4edf236 · 4edf236
2 parents 6a73ead + dc48864
commit 4edf236
Show file tree

Hide file tree

Showing 14 changed files with 80 additions and 192 deletions.
diff --git a/src/cache_handler.ts b/src/cache_handler.ts
@@ -9,7 +9,7 @@ import chunker from 'stream-chunker';
 import { Readable, Transform } from 'stream';
 import { Record, RecordPackage } from '@biothings-explorer/api-response-transform';
 import { threadId } from 'worker_threads';
-import MetaKG from '../../smartapi-kg/built';
+import MetaKG from '@biothings-explorer/smartapi-kg';
 import QEdge from './query_edge';
 import { QueryHandlerOptions } from '@biothings-explorer/types';
 
@@ -113,7 +113,7 @@ export default class CacheHandler {
   }
 
   async categorizeEdges(qEdges: QEdge[]): Promise<{ cachedRecords: Record[]; nonCachedQEdges: QEdge[] }> {
-    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === "true") {
+    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === 'true') {
       return {
         cachedRecords: [],
         nonCachedQEdges: qEdges,
@@ -210,7 +210,7 @@ export default class CacheHandler {
   }
 
   async cacheEdges(queryRecords: Record[]): Promise<void> {
-    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === "true") {
+    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === 'true') {
       if (global.parentPort) {
         global.parentPort.postMessage({ threadId, cacheDone: true });
       }
@@ -257,8 +257,8 @@ export default class CacheHandler {
                   resolve();
                 });
             });
-            if (process.env.QEDGE_CACHE_TIME_S !== "0") {
-                await redisClient.client.expireTimeout(redisID, process.env.QEDGE_CACHE_TIME_S || 1800);
+            if (process.env.QEDGE_CACHE_TIME_S !== '0') {
+              await redisClient.client.expireTimeout(redisID, process.env.QEDGE_CACHE_TIME_S || 1800);
             }
           } catch (error) {
             failedHashes.push(hash);

diff --git a/src/graph/graph.ts b/src/graph/graph.ts
@@ -4,8 +4,8 @@ import Debug from 'debug';
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
 import KGNode from './kg_node';
 import KGEdge from './kg_edge';
-import { Record } from '../../../api-response-transform/built';
-import { TrapiAuxiliaryGraph, TrapiResult } from '../types';
+import { Record } from '@biothings-explorer/api-response-transform';
+import { TrapiAuxiliaryGraph, TrapiResult } from '@biothings-explorer/types';
 import KnowledgeGraph from './knowledge_graph';
 const debug = Debug('bte:biothings-explorer-trapi:Graph');
 
@@ -104,6 +104,12 @@ export default class BTEGraph {
           .map((item) => {
             this.edges[recordHash].addAdditionalAttributes(item, record.mappedResponse[item]);
           });
+        if (record.knowledge_level) {
+          this.edges[recordHash].addAdditionalAttributes('biolink:knowledge_level', record.knowledge_level);
+        }
+        if (record.agent_type) {
+          this.edges[recordHash].addAdditionalAttributes('biolink:agent_type', record.agent_type);
+        }
         this.edges[recordHash].addSource(record.provenanceChain);
         Object.entries(record.qualifiers).forEach(([qualifierType, qualifier]) => {
           this.edges[recordHash].addQualifier(qualifierType, qualifier);

diff --git a/src/graph/kg_edge.ts b/src/graph/kg_edge.ts
@@ -1,5 +1,5 @@
-import { ProvenanceChainItem } from '@biothings-explorer/api-response-transform';
-import { TrapiAttribute } from '../types';
+import { TrapiSource } from '@biothings-explorer/types';
+import { TrapiAttribute } from '@biothings-explorer/types';
 
 export interface KGEdgeInfo {
   object: string;
@@ -20,6 +20,7 @@ export default class KGEdge {
         resource_id: string;
         resource_role: string;
         upstream_resource_ids?: Set<string>;
+        source_record_urls?: Set<string>;
       };
     };
   };
@@ -68,7 +69,7 @@ export default class KGEdge {
     });
   }
 
-  addSource(source: ProvenanceChainItem | ProvenanceChainItem[]): void {
+  addSource(source: TrapiSource | TrapiSource[]): void {
     if (typeof source === 'undefined') {
       return;
     }
@@ -77,19 +78,26 @@ export default class KGEdge {
     }
     source.forEach((item) => {
       if (!this.sources[item.resource_id]) this.sources[item.resource_id] = {};
+      if (item.upstream_resource_ids && !Array.isArray(item.upstream_resource_ids)) {
+        item.upstream_resource_ids = [item.upstream_resource_ids];
+      }
+      if (item.source_record_urls && !Array.isArray(item.source_record_urls)) {
+        item.source_record_urls = [item.source_record_urls];
+      }
       if (!this.sources[item.resource_id][item.resource_role]) {
         this.sources[item.resource_id][item.resource_role] = {
           resource_id: item.resource_id,
           resource_role: item.resource_role,
           upstream_resource_ids: item.upstream_resource_ids ? new Set(item.upstream_resource_ids) : undefined,
+          source_record_urls: item.source_record_urls ? new Set(item.source_record_urls) : undefined,
         };
       }
-      if (item.upstream_resource_ids && !Array.isArray(item.upstream_resource_ids)) {
-        item.upstream_resource_ids = [item.upstream_resource_ids];
-      }
       item.upstream_resource_ids?.forEach((upstream) =>
         this.sources[item.resource_id][item.resource_role].upstream_resource_ids.add(upstream),
       );
+      item.source_record_urls?.forEach((url) =>
+        this.sources[item.resource_id][item.resource_role].source_record_urls.add(url),
+      );
     });
   }
 

diff --git a/src/graph/kg_node.ts b/src/graph/kg_node.ts
@@ -1,4 +1,4 @@
-import { TrapiAttribute } from '../types';
+import { TrapiAttribute } from '@biothings-explorer/types';
 
 export interface KGNodeInfo {
   label: string;

diff --git a/src/graph/knowledge_graph.ts b/src/graph/knowledge_graph.ts
@@ -9,14 +9,16 @@ import {
   TrapiKGNodes,
   TrapiQualifier,
   TrapiSource,
-} from '../types';
+} from '@biothings-explorer/types';
 import KGNode from './kg_node';
 import KGEdge from './kg_edge';
 import { BTEGraphUpdate } from './graph';
 import { APIDefinition } from '@biothings-explorer/types';
 
 const debug = Debug('bte:biothings-explorer-trapi:KnowledgeGraph');
 
+const NON_ARRAY_ATTRIBUTES = ['biolink:knowledge_level', 'biolink:agent_type'];
+
 export default class KnowledgeGraph {
   nodes: {
     [nodePrimaryID: string]: TrapiKGNode;
@@ -114,11 +116,14 @@ export default class KnowledgeGraph {
     }
 
     Object.entries(kgEdge.attributes).forEach(([key, value]) => {
-      if (key == 'edge-attributes') return;
+      if (key === 'edge-attributes') return;
       // if (key == 'edge-attributes') return;
       attributes.push({
         attribute_type_id: key,
-        value: Array.from(value as Set<string>),
+        value: // technically works for numbers as well
+          NON_ARRAY_ATTRIBUTES.includes(key)
+            ? [...(value as Set<string>)].reduce((acc, val) => acc + val)
+            : Array.from(value as Set<string>),
         //value_type_id: 'bts:' + key,
       });
     });
@@ -137,6 +142,7 @@ export default class KnowledgeGraph {
         const trapiSource: TrapiSource = {
           ...sourceObj,
           upstream_resource_ids: sourceObj.upstream_resource_ids ? [...sourceObj.upstream_resource_ids] : undefined,
+          source_record_urls: sourceObj.source_record_urls ? [...sourceObj.source_record_urls] : undefined,
         };
         sources.push(trapiSource);
       });

diff --git a/src/index.ts b/src/index.ts
@@ -24,7 +24,7 @@ import {
   TrapiQueryGraph,
   TrapiResponse,
   TrapiResult,
-} from './types';
+} from '@biothings-explorer/types';
 import { QueryHandlerOptions } from '@biothings-explorer/types';
 import BTEGraph from './graph/graph';
 import QEdge from './query_edge';
@@ -173,6 +173,8 @@ export default class TRAPIQueryHandler {
         const source = Object.entries(ontologyKnowledgeSourceMapping).find(([prefix]) => {
           return expanded.includes(prefix);
         })[1];
+        subclassEdge.addAdditionalAttributes('biolink:knowledge_level', 'knowledge_assertion')
+        subclassEdge.addAdditionalAttributes('biolink:agent_type', 'manual_agent')
         subclassEdge.addSource([
           { resource_id: source, resource_role: 'primary_knowledge_source' },
           {
@@ -209,7 +211,7 @@ export default class TRAPIQueryHandler {
         suffix += 1;
       }
       const supportGraphID = `support${suffix}-${boundEdgeID}`;
-      auxGraphs[supportGraphID] = { edges: supportGraph };
+      auxGraphs[supportGraphID] = { edges: supportGraph, attributes: [] };
       if (!edgesIDsByAuxGraphID[supportGraphID]) {
         edgesIDsByAuxGraphID[supportGraphID] = new Set();
       }
@@ -221,6 +223,8 @@ export default class TRAPIQueryHandler {
           object: object,
         });
         boundEdge.addAdditionalAttributes('biolink:support_graphs', [supportGraphID]);
+        boundEdge.addAdditionalAttributes('biolink:knowledge_level', 'logical_entailment')
+        boundEdge.addAdditionalAttributes('biolink:agent_type', 'automated_agent')
         boundEdge.addSource([
           {
             resource_id: this.options.provenanceUsesServiceProvider
@@ -250,7 +254,7 @@ export default class TRAPIQueryHandler {
                     boundIDs.add(binding.id);
                   }
                 } else if (!boundIDs.has(nodesToRebind[binding.id].newNode)) {
-                  newBindings.push({ id: nodesToRebind[binding.id].newNode });
+                  newBindings.push({ id: nodesToRebind[binding.id].newNode, attributes: [] });
                   boundIDs.add(nodesToRebind[binding.id].newNode);
                 }
                 return { boundIDs, newBindings };
@@ -272,7 +276,7 @@ export default class TRAPIQueryHandler {
                     boundIDs.add(binding.id);
                   }
                 } else if (!boundIDs.has(edgesToRebind[binding.id])) {
-                  newBindings.push({ id: edgesToRebind[binding.id] });
+                  newBindings.push({ id: edgesToRebind[binding.id], attributes: [] });
                   boundIDs.add(edgesToRebind[binding.id]);
                   resultBoundEdgesWithAuxGraphs.add(edgesToRebind[binding.id]);
                 }
@@ -333,7 +337,7 @@ export default class TRAPIQueryHandler {
       description: `Query processed successfully, retrieved ${results.length} results.`,
       schema_version: global.SCHEMA_VERSION,
       biolink_version: global.BIOLINK_VERSION,
-      workflow: [{ id: 'lookup' }],
+      workflow: [{ id: this.options.smartAPIID || this.options.teamName ? 'lookup' : 'lookup_and_score' }],
       message: {
         query_graph: this.originalQueryGraph,
         knowledge_graph: this.knowledgeGraph.kg,

diff --git a/src/inferred_mode/inferred_mode.ts b/src/inferred_mode/inferred_mode.ts
@@ -6,9 +6,8 @@ import biolink from '../biolink';
 import { getTemplates, MatchedTemplate, TemplateLookup } from './template_lookup';
 import { scaled_sigmoid, inverse_scaled_sigmoid } from '../results_assembly/score';
 import TRAPIQueryHandler from '../index';
-import { QueryHandlerOptions } from '@biothings-explorer/types';
 import {
-  CompactQualifiers,
+  QueryHandlerOptions,
   TrapiAuxGraphCollection,
   TrapiEdgeBinding,
   TrapiKnowledgeGraph,
@@ -17,7 +16,8 @@ import {
   TrapiQueryGraph,
   TrapiResponse,
   TrapiResult,
-} from '../types';
+} from '@biothings-explorer/types';
+import { CompactQualifiers } from '../index';
 const debug = Debug('bte:biothings-explorer-trapi:inferred-mode');
 
 export interface CombinedResponse {
@@ -288,8 +288,8 @@ export default class InferredQueryHandler {
     newResponse.message.results.forEach((result) => {
       const translatedResult: TrapiResult = {
         node_bindings: {
-          [qEdge.subject]: [{ id: result.node_bindings.creativeQuerySubject[0].id }],
-          [qEdge.object]: [{ id: result.node_bindings.creativeQueryObject[0].id }],
+          [qEdge.subject]: [{ id: result.node_bindings.creativeQuerySubject[0].id, attributes: [] }],
+          [qEdge.object]: [{ id: result.node_bindings.creativeQueryObject[0].id, attributes: [] }],
         },
         pfocr: result.pfocr?.length ? result.pfocr : undefined,
         analyses: [
@@ -311,14 +311,14 @@ export default class InferredQueryHandler {
       // Direct edge answers stand on their own, not as an inferred edge.
       if (Object.keys(result.node_bindings).length == 2) {
         const boundEdgeID = Object.values(result.analyses[0].edge_bindings)[0][0].id;
-        translatedResult.analyses[0].edge_bindings = { [qEdgeID]: [{ id: boundEdgeID }] };
+        translatedResult.analyses[0].edge_bindings = { [qEdgeID]: [{ id: boundEdgeID, attributes: [] }] };
       } else {
         // Create an aux graph using the result and associate it with an inferred Edge
         const inferredEdgeID = `inferred-${resultCreativeSubjectID}-${qEdge.predicates[0].replace(
           'biolink:',
           '',
         )}-${resultCreativeObjectID}`;
-        translatedResult.analyses[0].edge_bindings = { [qEdgeID]: [{ id: inferredEdgeID }] };
+        translatedResult.analyses[0].edge_bindings = { [qEdgeID]: [{ id: inferredEdgeID, attributes: [] }] };
         if (!combinedResponse.message.knowledge_graph.edges[inferredEdgeID]) {
           combinedResponse.message.knowledge_graph.edges[inferredEdgeID] = {
             subject: resultCreativeSubjectID,
@@ -332,7 +332,11 @@ export default class InferredQueryHandler {
                 resource_role: 'primary_knowledge_source',
               },
             ],
-            attributes: [{ attribute_type_id: 'biolink:support_graphs', value: [] }],
+            attributes: [
+              { attribute_type_id: 'biolink:support_graphs', value: [] },
+              { attribute_type_id: 'biolink:knowledge_level', value: "prediction" },
+              { attribute_type_id: 'biolink:agent_type', value: "computational_model" },
+            ],
           };
         }
         let auxGraphSuffix = 0;
@@ -353,6 +357,7 @@ export default class InferredQueryHandler {
             },
             [] as string[],
           ),
+          attributes: []
         };
       }
 
@@ -385,9 +390,9 @@ export default class InferredQueryHandler {
         if (typeof combinedResponse.message.results[resultID].analyses[0].score !== 'undefined') {
           combinedResponse.message.results[resultID].analyses[0].score = resScore
             ? scaled_sigmoid(
-              inverse_scaled_sigmoid(combinedResponse.message.results[resultID].analyses[0].score) +
-              inverse_scaled_sigmoid(resScore),
-            )
+                inverse_scaled_sigmoid(combinedResponse.message.results[resultID].analyses[0].score) +
+                  inverse_scaled_sigmoid(resScore),
+              )
             : combinedResponse.message.results[resultID].analyses[0].score;
         } else {
           combinedResponse.message.results[resultID].analyses[0].score = resScore;
@@ -498,7 +503,7 @@ export default class InferredQueryHandler {
       description: '',
       schema_version: global.SCHEMA_VERSION,
       biolink_version: global.BIOLINK_VERSION,
-      workflow: [{ id: 'lookup' }],
+      workflow: [{ id: 'lookup_and_score' }],
       message: {
         query_graph: this.queryGraph,
         knowledge_graph: {
@@ -555,9 +560,11 @@ export default class InferredQueryHandler {
           const message = [
             `Addition of ${creativeLimitHit} results from Template ${i + 1}`,
             Object.keys(combinedResponse.message.results).length === this.CREATIVE_LIMIT ? ' meets ' : ' exceeds ',
-            `creative result maximum of ${this.CREATIVE_LIMIT} (reaching ${Object.keys(combinedResponse.message.results).length
+            `creative result maximum of ${this.CREATIVE_LIMIT} (reaching ${
+              Object.keys(combinedResponse.message.results).length
             } merged). `,
-            `Response will be truncated to top-scoring ${this.CREATIVE_LIMIT} results. Skipping remaining ${subQueries.length - (i + 1)
+            `Response will be truncated to top-scoring ${this.CREATIVE_LIMIT} results. Skipping remaining ${
+              subQueries.length - (i + 1)
             } `,
             subQueries.length - (i + 1) === 1 ? `template.` : `templates.`,
           ].join('');
@@ -582,8 +589,9 @@ export default class InferredQueryHandler {
       const total =
         Object.values(mergedResultsCount).reduce((sum, count) => sum + count, 0) +
         Object.keys(mergedResultsCount).length;
-      const message = `Merging Summary: (${total}) inferred-template results were merged into (${Object.keys(mergedResultsCount).length
-        }) final results, reducing result count by (${total - Object.keys(mergedResultsCount).length})`;
+      const message = `Merging Summary: (${total}) inferred-template results were merged into (${
+        Object.keys(mergedResultsCount).length
+      }) final results, reducing result count by (${total - Object.keys(mergedResultsCount).length})`;
       debug(message);
       combinedResponse.logs.push(new LogEntry('INFO', null, message).getLog());
     }

diff --git a/src/inferred_mode/template_lookup.ts b/src/inferred_mode/template_lookup.ts
@@ -2,7 +2,8 @@
 import { promises as fs } from 'fs';
 import path from 'path';
 import async from 'async';
-import { TrapiQueryGraph, CompactQualifiers } from '../types';
+import { TrapiQueryGraph } from '@biothings-explorer/types';
+import { CompactQualifiers } from '../types';
 
 export interface TemplateLookup {
   subject: string;

diff --git a/src/query_edge.ts b/src/query_edge.ts
@@ -6,7 +6,7 @@ import { Record, RecordNode, FrozenRecord } from '@biothings-explorer/api-respon
 import QNode from './query_node';
 import { QNodeInfo } from './query_node';
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
-import { TrapiAttributeConstraint, TrapiQualifierConstraint } from './types';
+import { TrapiAttributeConstraint, TrapiQualifierConstraint } from '@biothings-explorer/types';
 
 const debug = Debug('bte:biothings-explorer-trapi:QEdge');
 

diff --git a/src/query_graph.ts b/src/query_graph.ts
@@ -7,7 +7,7 @@ import biolink from './biolink';
 import { resolveSRI } from 'biomedical_id_resolver';
 import _ from 'lodash';
 import * as utils from './utils';
-import { TrapiQueryGraph } from './types';
+import { TrapiQueryGraph } from '@biothings-explorer/types';
 
 const debug = Debug('bte:biothings-explorer-trapi:query_graph');
 

diff --git a/src/results_assembly/pfocr.ts b/src/results_assembly/pfocr.ts
@@ -4,7 +4,7 @@ const debug = Debug('bte:biothings-explorer-trapi:pfocr');
 import { intersection } from '../utils';
 import _ from 'lodash';
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
-import { TrapiResult } from '../types';
+import { TrapiResult } from '@biothings-explorer/types';
 
 // the minimum acceptable intersection size between the CURIEs
 // in a TRAPI result and in a PFOCR figure.