feat: single-queue subquerying, per-subquery cache

This commit also contains a lot of refactoring and renaming, not keeping things particularly pretty given the planned migration to python down the line.

package.json

@@ -1,7 +1,7 @@
 {
-  "name": "@biothings-explorer/bte-trapi",
+  "name": "@retriever/main",
   "version": "3.0.0",
-  "description": "BioThings Explorer's TRAPI Knowledgebase Query API",
+  "description": "Retriever, a centralized subgraph-retrieval module",
   "main": "packages/server/built/server.js",
   "scripts": {
     "setup": "pnpm run clone && pnpm i",
@@ -26,29 +26,31 @@
     "start": "./scripts/start_server.sh",
     "stop": "./scripts/stop_server.sh",
     "watch": "nodemon -e js,mjs,json,ts --watch './packages/**/src/*' './packages/**/data/*' --exec 'pnpm run build && node .'",
-    "debug": "DEBUG=biomedical-id-resolver,bte* pnpm run watch"
+    "debug": "DEBUG=biomedical-id-resolver,retriever* pnpm run watch"
   },
   "engines": {
     "node": ">=18.0.0",
     "pnpm": ">=8.9.2"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/biothings/biothings_explorer.git"
+    "url": "git+https://github.com/BioPack-team/retriever.git"
   },
   "keywords": [
-    "bte",
+    "BioPack",
+    "retriever",
     "biothings",
+    "biopack",
     "single",
     "hop",
     "query"
   ],
-  "author": "BioThings Team",
+  "author": "BioPack Team",
   "license": "ISC",
   "bugs": {
-    "url": "https://github.com/biothings/biothings_explorer/issues"
+    "url": "https://github.com/BioPack-team/retriever/issues"
   },
-  "homepage": "https://github.com/biothings/biothings_explorer#readme",
+  "homepage": "https://github.com/BioPack-team/retriever#readme",
   "devDependencies": {
     "@types/node": "^20.8.7",
     "@typescript-eslint/eslint-plugin": "^6.8.0",
@@ -75,6 +77,7 @@
     "typescript": "5.2.2"
   },
   "dependencies": {
+    "@retriever/call-apis": "workspace:./packages/call-apis",
     "@bull-board/express": "^5.9.1",
     "@sentry/node": "^7.74.1",
     "@sentry/profiling-node": "^1.2.1",

packages/api-response-transform/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "@biothings-explorer/api-response-transform",
+  "name": "@retriever/api-response-transform",
   "version": "1.12.0",
   "description": "a NodeJS library to transform API responses into biolink model",
   "main": "built/index.js",
@@ -18,20 +18,22 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/biothings/api-respone-transform.js.git"
+    "url": "git+https://github.com/BioPack-team/retriever"
   },
   "keywords": [
+    "BioPack",
+    "retriever",
     "transform",
     "biolink",
     "api",
     "json"
   ],
-  "author": "BioThings Team",
+  "author": "BioPack Team",
   "license": "ISC",
   "bugs": {
-    "url": "https://github.com/biothings/api-respone-transform.js/issues"
+    "url": "https://github.com/BioPack-team/retriever/issues"
   },
-  "homepage": "https://github.com/biothings/api-respone-transform.js#readme",
+  "homepage": "https://github.com/BioPack-team/retriever#readme",
   "devDependencies": {
     "@babel/core": "^7.23.2",
     "@babel/preset-env": "^7.23.2",
@@ -58,7 +60,10 @@
     "typescript": "^5.2.2"
   },
   "dependencies": {
-    "@biothings-explorer/utils": "workspace:../utils",
+    "@retriever/utils": "workspace:../utils",
+    "@retriever/types": "workspace:../types",
+    "@retriever/graph": "workspace:../graph",
+    "@retriever/smartapi-kg": "workspace:../smartapi-kg",
     "@commitlint/cli": "^17.8.1",
     "@commitlint/config-conventional": "^11.0.0",
     "async": "^3.2.4",

packages/api-response-transform/src/index.ts

@@ -7,18 +7,17 @@ import BaseTransformer from "./transformers/transformer";
 import TRAPITransformer from "./transformers/trapi_transformer";
 import EBIProteinTransformer from "./transformers/ebi_protein_transformer";
 import JQTransformer from "./transformers/jq_transformer";
-import { BTEQueryObject } from "./types";
-import { Record } from "./record";
+import { RetrieverQueryObject } from "./types";
+import { Record } from "@retriever/graph";
 import Debug from "debug";
-const debug = Debug("bte:api-response-transform:index");
-export * from "./record";
+const debug = Debug("retriever:api-response-transform:index");
 export * from "./types";
 
 export default class Transformer {
-  private data: BTEQueryObject;
+  private data: RetrieverQueryObject;
   private tf: BaseTransformer;
   config: any;
-  constructor(data: BTEQueryObject, config: any) {
+  constructor(data: RetrieverQueryObject, config: any) {
     this.data = data;
     this.config = config;
     this.route();
@@ -46,12 +45,18 @@ export default class Transformer {
     } else if (api.startsWith("SEMMED")) {
       this.tf = new SemmedTransformer(this.data, this.config);
     } else if (api === "Monarch API") {
-      this.tf = new JQTransformer(this.data, { ...this.config, type: "monarch" });
+      this.tf = new JQTransformer(this.data, {
+        ...this.config,
+        type: "monarch",
+      });
     } else if (api === "EBI Proteins API") {
       // this.tf = new EBIProteinTransformer(this.data, this.config)
       this.tf = new JQTransformer(this.data, { ...this.config, type: "ebi" });
     } else if (tags.includes("biothings")) {
-      this.tf = new JQTransformer(this.data, { ...this.config, type: "biothings" });
+      this.tf = new JQTransformer(this.data, {
+        ...this.config,
+        type: "biothings",
+      });
     } else if (tags.includes("ctd")) {
       this.tf = new JQTransformer(this.data, { ...this.config, type: "ctd" });
     } else if (tags.includes("opentarget")) {

packages/api-response-transform/src/jq_utils.ts

@@ -1,7 +1,6 @@
-import { JQVariable, BTEKGOperationObject } from "./types";
+import { JQVariable } from "./types";
 import Path from "path";
 import fs from "fs";
-import { toArray } from "./utils";
 
 const functions = fs.readFileSync(
   Path.resolve(`${__dirname}/../data/jq/utils.jq`),

packages/api-response-transform/src/transformers/biothings_transformer.ts

@@ -5,10 +5,10 @@ export default class BioThingsTransformer extends BaseTransformer {
   async pairCurieWithAPIResponse() {
     if (this.edge.query_operation.method === "post") {
       let res = {};
-      const mapper = item => {
+      const mapper = (item: { query: string | string[]; }) => {
         // for input not found, BioThings API returns an entry with a key "notfound" equal to true
         if (!("notfound" in item)) {
-          let input = generateCurie(this.edge.association.input_id, item.query);
+          let input = generateCurie(this.edge.association.input_id as string, item.query);
           if (input in res) {
             res[input].push(item);
           } else {

packages/api-response-transform/src/transformers/transformer.ts

@@ -1,20 +1,19 @@
 import { transform } from "../json_transform/index";
 import { JSONDoc } from "../json_transform/types";
 import { generateCurie, toArray } from "../utils";
-import { BTEKGOperationObject, BTEQueryObject } from "../types";
-import { Record } from "../record";
-import { FrozenRecord } from "../record";
-import * as _ from "lodash";
+import { RetrieverQueryObject } from "../types";
+import _ from "lodash";
 import Debug from "debug";
 import async from "async";
-const debug = Debug("bte:api-response-transform:transformer");
+import { Record, APIEdge } from "@retriever/graph";
+const debug = Debug("retriever:api-response-transform:transformer");
 
 export default class BaseTransformer {
-  protected edge: BTEKGOperationObject;
-  protected data: BTEQueryObject;
+  protected edge: APIEdge;
+  protected data: RetrieverQueryObject;
   public config: any;
 
-  constructor(data: BTEQueryObject, config: any) {
+  constructor(data: RetrieverQueryObject, config: any) {
     this.data = data;
     this.edge = data.edge;
     this.config = config;
@@ -25,7 +24,7 @@ export default class BaseTransformer {
    */
   async pairCurieWithAPIResponse() {
     const input = generateCurie(
-      this.edge.association.input_id,
+      this.edge.association.input_id as string,
       this.edge.input.hasOwnProperty("queryInputs")
         ? this.edge.input["queryInputs"]
         : (this.edge.input as string),
@@ -228,7 +227,7 @@ export default class BaseTransformer {
 
   _removeNonEdgeData(mappedResponse: any) {
     delete mappedResponse["@type"];
-    delete mappedResponse[this.edge.association.output_id];
+    delete mappedResponse[this.edge.association.output_id as string];
     delete mappedResponse["input_name"];
     delete mappedResponse["output_name"];
     return mappedResponse;
@@ -298,7 +297,7 @@ export default class BaseTransformer {
             );
             await async.eachSeries(
               Object.entries(predicateResponse),
-              async ([predicate, mappedResponses]) => {
+              async ([_predicate, mappedResponses]) => {
                 if (
                   Array.isArray(mappedResponses) &&
                   mappedResponses.length > 0
@@ -330,7 +329,7 @@ export default class BaseTransformer {
    * @param {Object} mappedResponse - JSON response representing an output.
    */
   extractObjectIDs(mappedResponse: object) {
-    const output_id_type = this.edge.association.output_id;
+    const output_id_type = this.edge.association.output_id as string;
     if (!(output_id_type in mappedResponse)) {
       return [];
     }

packages/api-response-transform/src/transformers/trapi_transformer.ts

@@ -1,7 +1,7 @@
 import BaseTransformer from "./transformer";
-import { Record } from "../record";
+import { Record } from "@retriever/graph";
 import { JSONDoc } from "../json_transform/types";
-import { removeBioLinkPrefix } from "@biothings-explorer/utils";
+import { removeBioLinkPrefix } from "@retriever/utils";
 
 export default class TRAPITransformer extends BaseTransformer {
   _getUniqueEdges() {

packages/api-response-transform/src/types.ts

@@ -1,65 +1,24 @@
+import { APIEdge } from "@retriever/graph";
 import { JSONDoc } from "./json_transform/types";
+import { XBTEKGSOperationObject } from "@retriever/smartapi-kg";
 
-interface KGAssociationObject {
-  input_id?: string;
-  input_type: string;
-  output_id?: string;
-  output_type: string;
-  predicate: string;
-  source?: string;
-  api_name?: string;
-}
-
-interface XBTEParametersObject {
-  [key: string]: string | number;
-}
-
-interface TransformerObject {
+export interface TransformerObject {
   wrap?: string;
   pair?: string;
 }
 
-interface TransformerSet {
+export interface TransformerSet {
   [transformerPattern: string]: TransformerObject;
 }
 
-interface QueryOperationInterface {
-  path: string;
-  method: string;
-  server: string;
-  tags: string[];
-  path_params: string[];
-  params: XBTEParametersObject;
-  request_body: object;
-  supportBatch: boolean;
-  inputSeparator: string;
-  transformer: TransformerSet;
-}
-
-interface SmartAPIKGOperationObject {
-  association: KGAssociationObject;
-  query_operation?: QueryOperationInterface;
-  response_mapping?: any;
-  id?: string;
-  tags?: string[];
-}
-
-interface TemplatedInput {
+export interface TemplatedInput {
   queryInputs: string | string[];
   [additionalAttributes: string]: string | string[];
 }
 
-export interface BTEKGOperationObject extends SmartAPIKGOperationObject {
-  input: string | string[] | TemplatedInput;
-  reasoner_edge?: any;
-  filter?: string;
-  original_input?: object;
-  input_resolved_identifiers?: object;
-}
-
-export interface BTEQueryObject {
+export interface RetrieverQueryObject {
   response: JSONDoc | JSONDoc[] | { hits: JSONDoc[] };
-  edge: BTEKGOperationObject;
+  edge: APIEdge;
 }
 
 export interface JQVariable {

packages/api-response-transform/tsconfig.json

@@ -5,14 +5,40 @@
     "outDir": "./built",
     "rootDir": "./src",
     "paths": {
-      "@biothings-explorer/utils": ["../utils"]
+      "@retriever/utils": [
+        "../utils"
+      ],
+      "@retriever/types": [
+        "../types"
+      ],
+      "@retriever/graph": [
+        "../graph"
+      ],
+      "@retriever/smartapi_kg": [
+        "../smartapi-kg"
+      ]
     }
   },
-  "include": ["./src/**/*"],
-  "exclude": ["node_modules", "__tests__/"],
+  "include": [
+    "./src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "__tests__/"
+  ],
   "references": [
     {
       "path": "../utils"
+    },
+    {
+      "path": "../types"
+    },
+    {
+      "path": "../graph"
+    },
+    {
+      "path": "../smartapi-kg"
     }
+
   ]
 }

packages/biolink-model/package.json

@@ -17,20 +17,21 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/biothings/biolink-model.js.git"
+    "url": "git+https://github.com/BioPack-team/retriever"
   },
   "keywords": [
-    "biothings-explorer",
+    "BioPack",
+    "retriever",
     "biolink",
     "biomedical",
     "ontology"
   ],
   "author": "jiwen xin",
   "license": "ISC",
   "bugs": {
-    "url": "https://github.com/biothings/biolink-model.js/issues"
+    "url": "https://github.com/BioPack-team/retriever/issues"
   },
-  "homepage": "https://github.com/biothings/biolink-model.js#readme",
+  "homepage": "https://github.com/BioPack-team/retriever#readme",
   "devDependencies": {
     "@types/debug": "^4.1.10",
     "@types/jest": "^26.0.24",

packages/biolink-model/src/loader/url_loader.ts

@@ -7,7 +7,7 @@ export default class URLLoader extends Loader {
   async load(input) {
     let res;
     try {
-      const userAgent = `BTE/${process.env.NODE_ENV === 'production' ? 'prod' : 'dev'} Node/${process.version} ${
+      const userAgent = `Retriever/${process.env.NODE_ENV === 'production' ? 'prod' : 'dev'} Node/${process.version} ${
         process.platform
       }`;
       res = await axios.get(input, {