Merge pull request #77 from biothings/monarch-pagination

Monarch pagination
biothings · Aug 20, 2024 · 4c59a93 · 4c59a93
2 parents 5616ad4 + f084091
commit 4c59a93
Show file tree

Hide file tree

Showing 5 changed files with 67 additions and 18 deletions.
diff --git a/src/builder/base_query_builder.ts b/src/builder/base_query_builder.ts
@@ -85,9 +85,9 @@ export default class BaseQueryBuilder {
     return config;
   }
 
-  needPagination(apiResponse: unknown): number {
+  needPagination(apiResponse: unknown): {paginationStart: number, paginationSize: number} {
     // implemented in subclasses
-    return 0;
+    return {paginationSize: 0, paginationStart: 0};
   }
 
   getNext(): AxiosRequestConfig {

diff --git a/src/builder/query_builder.ts b/src/builder/query_builder.ts
@@ -106,7 +106,7 @@ export default class QueryBuilder extends BaseQueryBuilder {
     return config;
   }
 
-  needPagination(apiResponse: unknown): number {
+  needPagination(apiResponse: unknown): {paginationStart: number, paginationSize: number}  {
     if (
       this.APIEdge.query_operation.method === "get" &&
       this.APIEdge.tags.includes("biothings")
@@ -117,12 +117,12 @@ export default class QueryBuilder extends BaseQueryBuilder {
       ) {
         if (this.start + (apiResponse as BiothingsResponse).hits.length < 10000) {
           this.hasNext = true;
-          return this.start + 1000;
+          return {paginationStart: this.start + 1000, paginationSize: 1000};
         }
       }
     }
     this.hasNext = false;
-    return 0;
+    return {paginationStart: 0, paginationSize: 0};
   }
 
   getNext(): AxiosRequestConfig {

diff --git a/src/builder/template_query_builder.ts b/src/builder/template_query_builder.ts
@@ -37,8 +37,12 @@ export default class TemplateQueryBuilder extends BaseQueryBuilder {
   /**
    * Construct input based on method and inputSeparator
    */
-  _getInput(APIEdge: APIEdge): string | string[] {
-    return APIEdge.input as string | string[];
+  _getInput(APIEdge: APIEdge): TemplatedInput {
+    let baseInput = APIEdge.input as TemplatedInput;
+    if (this.APIEdge.query_operation.paginated) {
+      (baseInput as any).start = this.start;
+    }
+    return baseInput;
   }
 
   /**
@@ -120,15 +124,34 @@ export default class TemplateQueryBuilder extends BaseQueryBuilder {
     return config;
   }
 
-  needPagination(apiResponse: unknown): number {
+  // util for pagination
+  _getDescendantProp(obj: any, desc: string): any {
+    let arr = desc.split(".");
+    for (let i = 0; i < arr.length; i++) {
+      obj = obj?.[arr[i]];
+    }
+    return obj;
+  }
+
+  needPagination(apiResponse: unknown): {paginationStart: number, paginationSize: number}  {
+    if (this.APIEdge.query_operation.paginated) {
+      let resCount = this._getDescendantProp(apiResponse, this.APIEdge.query_operation.paginationData.countField);
+      if (Array.isArray(resCount)) resCount = resCount.length;
+      let resTotal = this._getDescendantProp(apiResponse, this.APIEdge.query_operation.paginationData.totalField);
+      let paginationSize = this.APIEdge.query_operation.paginationData.pageSize;
+      if (resTotal > this.start + resCount) {
+        this.hasNext = true;
+        return {paginationStart: this.start + paginationSize, paginationSize};
+      }
+    }
     // TODO check for biothings pending, use smarter post method (also do config properly to use new parameter)
-    if (
+    else if (
       this.APIEdge.query_operation.method === "post" &&
       this.APIEdge.tags.includes("biothings")
     ) {
       if ((apiResponse as BiothingsResponse).max_total > this.start + 1000) {
         this.hasNext = true;
-        return this.start + 1000;
+        return {paginationStart: this.start + 1000, paginationSize: 1000};
       }
     } else if (
       this.APIEdge.query_operation.method === "get" &&
@@ -140,23 +163,49 @@ export default class TemplateQueryBuilder extends BaseQueryBuilder {
       ) {
         if (this.start + (apiResponse as BiothingsResponse).hits.length < 10000) {
           this.hasNext = true;
-          return this.start + 1000;
+          return {paginationStart: this.start + 1000, paginationSize: 1000};
+        }
+      }
+    } else if (
+      this.APIEdge.query_operation.method === "get" &&
+      this.APIEdge.association.api_name === "Monarch API"
+    ) {
+      if (
+        (apiResponse as any).total >
+        this.start + (apiResponse as any).items.length
+      ) {
+        if (this.start + (apiResponse as any).items.length < 10000) {
+          this.hasNext = true;
+          return {paginationStart: this.start + 500, paginationSize: 500};
         }
       }
     }
     this.hasNext = false;
-    return 0;
+    return {paginationStart: 0, paginationSize: 0};
   }
 
   getNext(): AxiosRequestConfig {
+    if (this.APIEdge.query_operation.paginated) {
+      this.start += this.APIEdge.query_operation.paginationData.pageSize;
+      this.config = this.constructAxiosRequestConfig();
+      return this.config;
+    }
     const config = this.constructAxiosRequestConfig();
     if (
       this.APIEdge.query_operation.method === "post" &&
       this.APIEdge.tags.includes("biothings")
     ) {
       this.start = this.start + 1000;
       config.params.from = this.start;
-    } else {
+    } 
+    else if (
+      this.APIEdge.query_operation.method === "get" &&
+      this.APIEdge.association.api_name === "Monarch API"
+    ) {
+      this.start = this.start + 500;
+      config.params.offset = this.start;
+    }
+    else {
       this.start = Math.min(this.start + 1000, 9999);
       config.params.from = this.start;
       if (config.params.size + this.start > 10000) {

diff --git a/src/builder/trapi_query_builder.ts b/src/builder/trapi_query_builder.ts
@@ -104,9 +104,9 @@ export default class TRAPIQueryBuilder extends BaseQueryBuilder {
     return config;
   }
 
-  needPagination(apiResponse: TrapiResponse): number {
+  needPagination(apiResponse: TrapiResponse): {paginationStart: number, paginationSize: number}  {
     this.hasNext = false;
-    return 0;
+    return {paginationStart: 0, paginationSize: 0};
   }
 
   getNext(): AxiosRequestConfig {

diff --git a/src/query_pool.ts b/src/query_pool.ts
@@ -119,7 +119,7 @@ export default class APIQueryPool {
           "ERROR",
           null,
           `${(
-            error as Error
+            (error as Error).stack
           ).toString()} while configuring query. Query dump: ${JSON.stringify(
             query,
           )}`,
@@ -223,12 +223,12 @@ export default class APIQueryPool {
         edge: query.APIEdge,
       };
 
-      const queryNeedsPagination = query.needPagination(
+      const {paginationStart: queryNeedsPagination, paginationSize} = query.needPagination(
         unTransformedHits.response,
       );
       if (queryNeedsPagination) {
         const log = `Query requires pagination, will re-query to window ${queryNeedsPagination}-${
-          queryNeedsPagination + 1000
+          queryNeedsPagination + paginationSize
         }: ${query.APIEdge.query_operation.server} (${nInputs} ID${
           nInputs > 1 ? "s" : ""
         })`;