Support source_record_urls

src/cache_handler.ts

@@ -9,7 +9,7 @@ import chunker from 'stream-chunker';
 import { Readable, Transform } from 'stream';
 import { Record, RecordPackage } from '@biothings-explorer/api-response-transform';
 import { threadId } from 'worker_threads';
-import MetaKG from '../../smartapi-kg/built';
+import MetaKG from '@biothings-explorer/smartapi-kg';
 import QEdge from './query_edge';
 import { QueryHandlerOptions } from '@biothings-explorer/types';
 
@@ -113,7 +113,7 @@ export default class CacheHandler {
   }
 
   async categorizeEdges(qEdges: QEdge[]): Promise<{ cachedRecords: Record[]; nonCachedQEdges: QEdge[] }> {
-    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === "true") {
+    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === 'true') {
       return {
         cachedRecords: [],
         nonCachedQEdges: qEdges,
@@ -210,7 +210,7 @@ export default class CacheHandler {
   }
 
   async cacheEdges(queryRecords: Record[]): Promise<void> {
-    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === "true") {
+    if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS === 'true') {
       if (global.parentPort) {
         global.parentPort.postMessage({ threadId, cacheDone: true });
       }
@@ -257,8 +257,8 @@ export default class CacheHandler {
                   resolve();
                 });
             });
-            if (process.env.QEDGE_CACHE_TIME_S !== "0") {
-                await redisClient.client.expireTimeout(redisID, process.env.QEDGE_CACHE_TIME_S || 1800);
+            if (process.env.QEDGE_CACHE_TIME_S !== '0') {
+              await redisClient.client.expireTimeout(redisID, process.env.QEDGE_CACHE_TIME_S || 1800);
             }
           } catch (error) {
             failedHashes.push(hash);

src/graph/graph.ts

@@ -4,8 +4,8 @@ import Debug from 'debug';
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
 import KGNode from './kg_node';
 import KGEdge from './kg_edge';
-import { Record } from '../../../api-response-transform/built';
-import { TrapiAuxiliaryGraph, TrapiResult } from '../types';
+import { Record } from '@biothings-explorer/api-response-transform';
+import { TrapiAuxiliaryGraph, TrapiResult } from '@biothings-explorer/types';
 import KnowledgeGraph from './knowledge_graph';
 const debug = Debug('bte:biothings-explorer-trapi:Graph');
 

src/graph/kg_edge.ts

@@ -1,5 +1,5 @@
-import { ProvenanceChainItem } from '@biothings-explorer/api-response-transform';
-import { TrapiAttribute } from '../types';
+import { TrapiSource } from '@biothings-explorer/types';
+import { TrapiAttribute } from '@biothings-explorer/types';
 
 export interface KGEdgeInfo {
   object: string;
@@ -20,6 +20,7 @@ export default class KGEdge {
         resource_id: string;
         resource_role: string;
         upstream_resource_ids?: Set<string>;
+        source_record_urls?: Set<string>;
       };
     };
   };
@@ -68,7 +69,7 @@ export default class KGEdge {
     });
   }
 
-  addSource(source: ProvenanceChainItem | ProvenanceChainItem[]): void {
+  addSource(source: TrapiSource | TrapiSource[]): void {
     if (typeof source === 'undefined') {
       return;
     }
@@ -77,19 +78,26 @@ export default class KGEdge {
     }
     source.forEach((item) => {
       if (!this.sources[item.resource_id]) this.sources[item.resource_id] = {};
+      if (item.upstream_resource_ids && !Array.isArray(item.upstream_resource_ids)) {
+        item.upstream_resource_ids = [item.upstream_resource_ids];
+      }
+      if (item.source_record_urls && !Array.isArray(item.source_record_urls)) {
+        item.source_record_urls = [item.source_record_urls];
+      }
       if (!this.sources[item.resource_id][item.resource_role]) {
         this.sources[item.resource_id][item.resource_role] = {
           resource_id: item.resource_id,
           resource_role: item.resource_role,
           upstream_resource_ids: item.upstream_resource_ids ? new Set(item.upstream_resource_ids) : undefined,
+          source_record_urls: item.source_record_urls ? new Set(item.source_record_urls) : undefined,
         };
       }
-      if (item.upstream_resource_ids && !Array.isArray(item.upstream_resource_ids)) {
-        item.upstream_resource_ids = [item.upstream_resource_ids];
-      }
       item.upstream_resource_ids?.forEach((upstream) =>
         this.sources[item.resource_id][item.resource_role].upstream_resource_ids.add(upstream),
       );
+      item.source_record_urls?.forEach((url) =>
+        this.sources[item.resource_id][item.resource_role].source_record_urls.add(url),
+      );
     });
   }
 

src/graph/kg_node.ts

@@ -1,4 +1,4 @@
-import { TrapiAttribute } from '../types';
+import { TrapiAttribute } from '@biothings-explorer/types';
 
 export interface KGNodeInfo {
   label: string;

src/graph/knowledge_graph.ts

@@ -9,7 +9,7 @@ import {
   TrapiKGNodes,
   TrapiQualifier,
   TrapiSource,
-} from '../types';
+} from '@biothings-explorer/types';
 import KGNode from './kg_node';
 import KGEdge from './kg_edge';
 import { BTEGraphUpdate } from './graph';
@@ -137,6 +137,7 @@ export default class KnowledgeGraph {
         const trapiSource: TrapiSource = {
           ...sourceObj,
           upstream_resource_ids: sourceObj.upstream_resource_ids ? [...sourceObj.upstream_resource_ids] : undefined,
+          source_record_urls: sourceObj.source_record_urls ? [...sourceObj.source_record_urls] : undefined,
         };
         sources.push(trapiSource);
       });

src/index.ts

@@ -24,7 +24,7 @@ import {
   TrapiQueryGraph,
   TrapiResponse,
   TrapiResult,
-} from './types';
+} from '@biothings-explorer/types';
 import { QueryHandlerOptions } from '@biothings-explorer/types';
 import BTEGraph from './graph/graph';
 import QEdge from './query_edge';

src/inferred_mode/inferred_mode.ts

@@ -6,9 +6,8 @@ import biolink from '../biolink';
 import { getTemplates, MatchedTemplate, TemplateLookup } from './template_lookup';
 import { scaled_sigmoid, inverse_scaled_sigmoid } from '../results_assembly/score';
 import TRAPIQueryHandler from '../index';
-import { QueryHandlerOptions } from '@biothings-explorer/types';
 import {
-  CompactQualifiers,
+  QueryHandlerOptions,
   TrapiAuxGraphCollection,
   TrapiEdgeBinding,
   TrapiKnowledgeGraph,
@@ -17,7 +16,8 @@ import {
   TrapiQueryGraph,
   TrapiResponse,
   TrapiResult,
-} from '../types';
+} from '@biothings-explorer/types';
+import { CompactQualifiers } from '../index';
 const debug = Debug('bte:biothings-explorer-trapi:inferred-mode');
 
 export interface CombinedResponse {
@@ -385,9 +385,9 @@ export default class InferredQueryHandler {
         if (typeof combinedResponse.message.results[resultID].analyses[0].score !== 'undefined') {
           combinedResponse.message.results[resultID].analyses[0].score = resScore
             ? scaled_sigmoid(
-              inverse_scaled_sigmoid(combinedResponse.message.results[resultID].analyses[0].score) +
-              inverse_scaled_sigmoid(resScore),
-            )
+                inverse_scaled_sigmoid(combinedResponse.message.results[resultID].analyses[0].score) +
+                  inverse_scaled_sigmoid(resScore),
+              )
             : combinedResponse.message.results[resultID].analyses[0].score;
         } else {
           combinedResponse.message.results[resultID].analyses[0].score = resScore;
@@ -555,9 +555,11 @@ export default class InferredQueryHandler {
           const message = [
             `Addition of ${creativeLimitHit} results from Template ${i + 1}`,
             Object.keys(combinedResponse.message.results).length === this.CREATIVE_LIMIT ? ' meets ' : ' exceeds ',
-            `creative result maximum of ${this.CREATIVE_LIMIT} (reaching ${Object.keys(combinedResponse.message.results).length
+            `creative result maximum of ${this.CREATIVE_LIMIT} (reaching ${
+              Object.keys(combinedResponse.message.results).length
             } merged). `,
-            `Response will be truncated to top-scoring ${this.CREATIVE_LIMIT} results. Skipping remaining ${subQueries.length - (i + 1)
+            `Response will be truncated to top-scoring ${this.CREATIVE_LIMIT} results. Skipping remaining ${
+              subQueries.length - (i + 1)
             } `,
             subQueries.length - (i + 1) === 1 ? `template.` : `templates.`,
           ].join('');
@@ -582,8 +584,9 @@ export default class InferredQueryHandler {
       const total =
         Object.values(mergedResultsCount).reduce((sum, count) => sum + count, 0) +
         Object.keys(mergedResultsCount).length;
-      const message = `Merging Summary: (${total}) inferred-template results were merged into (${Object.keys(mergedResultsCount).length
-        }) final results, reducing result count by (${total - Object.keys(mergedResultsCount).length})`;
+      const message = `Merging Summary: (${total}) inferred-template results were merged into (${
+        Object.keys(mergedResultsCount).length
+      }) final results, reducing result count by (${total - Object.keys(mergedResultsCount).length})`;
       debug(message);
       combinedResponse.logs.push(new LogEntry('INFO', null, message).getLog());
     }

src/inferred_mode/template_lookup.ts

@@ -2,7 +2,8 @@
 import { promises as fs } from 'fs';
 import path from 'path';
 import async from 'async';
-import { TrapiQueryGraph, CompactQualifiers } from '../types';
+import { TrapiQueryGraph } from '@biothings-explorer/types';
+import { CompactQualifiers } from '../types';
 
 export interface TemplateLookup {
   subject: string;

src/query_edge.ts

@@ -6,7 +6,7 @@ import { Record, RecordNode, FrozenRecord } from '@biothings-explorer/api-respon
 import QNode from './query_node';
 import { QNodeInfo } from './query_node';
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
-import { TrapiAttributeConstraint, TrapiQualifierConstraint } from './types';
+import { TrapiAttributeConstraint, TrapiQualifierConstraint } from '@biothings-explorer/types';
 
 const debug = Debug('bte:biothings-explorer-trapi:QEdge');
 

src/query_graph.ts

@@ -7,7 +7,7 @@ import biolink from './biolink';
 import { resolveSRI } from 'biomedical_id_resolver';
 import _ from 'lodash';
 import * as utils from './utils';
-import { TrapiQueryGraph } from './types';
+import { TrapiQueryGraph } from '@biothings-explorer/types';
 
 const debug = Debug('bte:biothings-explorer-trapi:query_graph');
 

src/results_assembly/pfocr.ts

@@ -4,7 +4,7 @@ const debug = Debug('bte:biothings-explorer-trapi:pfocr');
 import { intersection } from '../utils';
 import _ from 'lodash';
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
-import { TrapiResult } from '../types';
+import { TrapiResult } from '@biothings-explorer/types';
 
 // the minimum acceptable intersection size between the CURIEs
 // in a TRAPI result and in a PFOCR figure.

src/results_assembly/query_results.ts

@@ -1,5 +1,5 @@
 import { LogEntry, StampedLog } from '@biothings-explorer/utils';
-import { TrapiResult } from '../types';
+import { TrapiResult } from '@biothings-explorer/types';
 import Debug from 'debug';
 import { zip } from 'lodash';
 const debug = Debug('bte:biothings-explorer-trapi:QueryResult');

src/types.ts

@@ -1,153 +1,5 @@
 import { TrapiLog } from '@biothings-explorer/utils';
 
-export interface TrapiQNode {
-  ids?: string[];
-  categories?: string[];
-  is_set?: boolean;
-  constraints?: TrapiAttributeConstraint[];
-}
-
-export interface TrapiQEdge {
-  knowledge_type?: string;
-  predicates?: string[];
-  subject: string;
-  object: string;
-  attribute_constraints?: TrapiAttributeConstraint[];
-  qualifier_constraints?: TrapiQualifierConstraint[];
-}
-
-export interface TrapiQueryGraph {
-  nodes: {
-    [QNodeID: string]: TrapiQNode;
-  };
-  edges: {
-    [QEdgeID: string]: TrapiQEdge;
-  };
-}
-
-export interface TrapiSource {
-  resource_id: string;
-  resource_role: string;
-  upstream_resource_ids?: string[];
-}
-
-export interface TrapiKGNodes {
-  [nodeID: string]: TrapiKGNode;
-}
-
-export interface TrapiKGEdges {
-  [edgeID: string]: TrapiKGEdge;
-}
-
-export interface TrapiKnowledgeGraph {
-  nodes: TrapiKGNodes;
-  edges: TrapiKGEdges;
-}
-
-export interface TrapiKGEdge {
-  predicate: string;
-  subject: string;
-  object: string;
-  attributes?: TrapiAttribute[];
-  qualifiers?: TrapiQualifier[];
-  sources: TrapiSource[];
-}
-
-export interface TrapiKGNode {
-  categories: string[];
-  name: string;
-  attributes?: TrapiAttribute[];
-}
-
-export interface TrapiAttribute {
-  attribute_type_id: string;
-  original_attribute_name?: string;
-  value: string | string[] | number | number[];
-  value_type_id?: string;
-  attribute_source?: string | null;
-  value_url?: string | null;
-  attributes?: TrapiAttribute;
-  [additionalProperties: string]: string | string[] | null | TrapiAttribute | number | number[];
-}
-
-export interface TrapiQualifier {
-  qualifier_type_id: string;
-  qualifier_value: string | string[];
-}
-
-export interface TrapiQualifierConstraint {
-  qualifier_set: TrapiQualifier[];
-}
-
-export interface TrapiAttributeConstraint {
-  id: string;
-  name: string;
-  not: boolean;
-  operator: string;
-  value: string | string[] | number | number[];
-}
-
-export interface TrapiNodeBinding {
-  id: string;
-  query_id?: string;
-  attributes?: TrapiAttribute[];
-}
-
-export interface TrapiEdgeBinding {
-  id: string;
-  attributes?: TrapiAttribute[];
-}
-
-export interface TrapiAnalysis {
-  resource_id?: string;
-  score?: number;
-  edge_bindings: {
-    [qEdgeID: string]: TrapiEdgeBinding[];
-  };
-  support_graphs?: string[];
-  scoring_method?: string;
-  attributes?: TrapiAttribute[];
-}
-
-export interface TrapiAuxiliaryGraph {
-  edges: string[];
-  attributes?: TrapiAttribute[];
-}
-
-export interface TrapiPfocrFigure {
-  figureUrl: string;
-  pmc: string;
-  matchedCuries: string[];
-  score: number;
-}
-
-export interface TrapiResult {
-  node_bindings: {
-    [qNodeID: string]: TrapiNodeBinding[];
-  };
-  analyses: TrapiAnalysis[];
-  pfocr?: TrapiPfocrFigure[];
-}
-
-export interface TrapiAuxGraphCollection {
-  [supportGraphID: string]: TrapiAuxiliaryGraph;
-}
-
-export interface TrapiResponse {
-  description?: string;
-  schema_version?: string;
-  biolink_version?: string;
-  workflow?: { id: string }[];
-  message: {
-    query_graph: TrapiQueryGraph;
-    knowledge_graph: TrapiKnowledgeGraph;
-    auxiliary_graphs?: TrapiAuxGraphCollection;
-    results: TrapiResult[];
-  };
-  logs: TrapiLog[];
-}
-
-
 export interface UnavailableAPITracker {
   [server: string]: { skip: boolean; skippedQueries: number };
 }