feat: CQDG-835 add sets patante

src/app.test.ts

@@ -1,16 +1,17 @@
-import { createSet, deleteSet, getSets, SubActionTypes, updateSetContent, updateSetTag } from '@ferlab/next/lib/sets';
-import { Set, UpdateSetContentBody, UpdateSetTagBody } from '@ferlab/next/lib/sets/types';
 import { jest } from '@jest/globals';
 import { Express } from 'express';
 import Keycloak from 'keycloak-connect';
 import request from 'supertest';
 
+import { createSet, deleteSet, getSets, SubActionTypes, updateSetContent, updateSetTag } from '#src/services/sets';
+import { Set, UpdateSetContentBody, UpdateSetTagBody } from '#src/services/sets/types';
+
 import buildApp from './app';
 import { keycloakClient, keycloakRealm, keycloakURL } from './config/env';
 import { getStatistics, Statistics } from './endpoints/statistics';
 import { getToken, publicKey } from './utils/authTestUtils';
 
-jest.mock('@ferlab/next/lib/sets/index');
+jest.mock('#src/services/sets/index');
 jest.mock('./endpoints/statistics');
 
 //todo: fix tests

src/app.ts

@@ -1,4 +1,3 @@
-import { SetSqon } from '@ferlab/next/lib/sets/types';
 import resolveSetIdMiddleware from '@ferlab/next/lib/sqon/resolveSetIdMiddleware';
 import compression from 'compression';
 import cors from 'cors';
@@ -7,6 +6,8 @@ import { StatusCodes } from 'http-status-codes';
 import { Keycloak } from 'keycloak-connect';
 import NodeCache from 'node-cache';
 
+import { SetSqon } from '#src/services/sets/types';
+
 import packageJson from '../package.json' assert { type: 'json' };
 import { cacheTTL, esHost, keycloakURL, usersApiURL } from './config/env';
 import { getExtendedMapping } from './endpoints/extendedMapping';

src/endpoints/phenotypes.ts

@@ -1,14 +1,13 @@
-import { SetSqon } from '@ferlab/next/lib/sets/types';
 import searchSqon from '@ferlab/next/lib/sqon/searchSqon';
 import { replaceSetByIds } from '@ferlab/next/lib/sqon/setSqon';
 import get from 'lodash/get';
 
+import { participantBiospecimenKey, participantFileKey, participantKey } from '#src/config/env';
+import { maxSetContentSize, participantIdKey, usersApiURL } from '#src/config/env';
+import runQuery from '#src/graphql/runQuery';
 import schema from '#src/graphql/schema';
 import esClient from '#src/services/elasticsearch/client';
-
-import { participantBiospecimenKey, participantFileKey, participantKey } from '../config/env';
-import { maxSetContentSize, participantIdKey, usersApiURL } from '../config/env';
-import runQuery from '../graphql/runQuery';
+import { SetSqon } from '#src/services/sets/types';
 
 const getPathToParticipantId = (type: string) => {
   if (type === 'biospecimen') {

src/graphql/runQuery.ts

@@ -1,8 +1,9 @@
-import { SetSqon, Sort } from '@ferlab/next/lib/sets/types';
 import { graphql } from 'graphql';
 import { ExecutionResult } from 'graphql/execution/execute';
 
-import esClient from '../services/elasticsearch/client';
+import esClient from '#src/services/elasticsearch/client';
+import { SetSqon, Sort } from '#src/services/sets/types';
+
 import schema from './schema';
 
 interface IrunQuery {

src/routes/sets/index.ts

@@ -1,10 +1,10 @@
-import { createSet, deleteSet, getSets, SubActionTypes, updateSetContent, updateSetTag } from '@ferlab/next/lib/sets';
-import { CreateSetBody, Set, UpdateSetContentBody, UpdateSetTagBody } from '@ferlab/next/lib/sets/types';
 import express from 'express';
 
 import { maxSetContentSize, usersApiURL } from '#src/config/env';
 import schema from '#src/graphql/schema';
 import esClient from '#src/services/elasticsearch/client';
+import { createSet, deleteSet, getSets, SubActionTypes, updateSetContent, updateSetTag } from '#src/services/sets';
+import { CreateSetBody, Set, UpdateSetContentBody, UpdateSetTagBody } from '#src/services/sets/types';
 
 const router = express.Router();
 

src/services/elasticsearch/utils.ts

@@ -32,3 +32,26 @@ export const getBody = ({ field, value, path, nested = false }) => {
     query: { bool: { must } },
   };
 };
+
+/**
+ * Calls the `search` on the given elasticsearch.Client to get a single page of results.
+ * @param {Object} esClient - an elasticsearch.Client object.
+ * @param {String} index - the name of the index (or alias) on which to search.
+ * @param {Object} query - an object containing the query,
+ */
+export const executeSearch = async (esClient, index, query) => {
+  const searchParams = {
+    index,
+    body: {
+      ...query,
+      size: typeof query.size === 'number' ? query.size : 0,
+    },
+  };
+
+  try {
+    return esClient.search(searchParams);
+  } catch (err) {
+    console.error(`Error searching ES with params ${JSON.stringify(searchParams)}`, err);
+    throw err;
+  }
+};

src/services/sets/getFamilyIds.ts

@@ -0,0 +1,89 @@
+import { Client } from '@opensearch-project/opensearch';
+
+import { esFileIndex, maxSetContentSize } from '#src/config/env';
+
+import { executeSearch } from '../elasticsearch/utils';
+
+interface IFileInfo {
+  data_type: string;
+  family_id: string;
+}
+
+/** Get IFileInfo: files data_types and family_ids */
+const getFilesInfo = async (fileIds: string[], es: Client): Promise<IFileInfo[]> => {
+  const esRequest = {
+    query: { bool: { must: [{ terms: { file_id: fileIds, boost: 0 } }] } },
+    _source: ['file_id', 'data_type', 'participants.family_id'],
+    sort: [{ data_type: { order: 'asc' } }],
+    size: maxSetContentSize,
+  };
+  const results = await executeSearch(es, esFileIndex, esRequest);
+  const hits = results?.body?.hits?.hits || [];
+  const sources = hits.map((hit) => hit._source);
+  const filesInfos = [];
+  sources?.forEach((source) => {
+    source.participants?.forEach((participant) => {
+      if (
+        participant.family_id &&
+        !filesInfos.find((f) => f.family_id === participant.family_id && f.data_type === source.data_type)
+      ) {
+        filesInfos.push({
+          data_type: source.data_type,
+          family_id: participant.family_id || '',
+        });
+      }
+    });
+  });
+  return filesInfos;
+};
+
+/** for each filesInfos iteration, get files from file.participants.family_id and file.data_type */
+const getFilesIdsMatched = async (filesInfos: IFileInfo[], es: Client): Promise<string[]> => {
+  const filesIdsMatched = [];
+  const results = await Promise.all(
+    filesInfos.map((info) => {
+      const esRequest = {
+        query: {
+          bool: {
+            must: [
+              { terms: { data_type: [info.data_type], boost: 0 } },
+              {
+                nested: {
+                  path: 'participants',
+                  query: { bool: { must: [{ match: { ['participants.family_id']: info.family_id } }] } },
+                },
+              },
+            ],
+          },
+        },
+        _source: ['file_id'],
+        size: maxSetContentSize,
+      };
+      return executeSearch(es, esFileIndex, esRequest);
+    })
+  );
+
+  for (const res of results) {
+    const hits = res?.body?.hits?.hits || [];
+    const sources = hits.map((hit) => hit._source);
+    filesIdsMatched.push(...sources.map((s) => s.file_id));
+  }
+
+  return filesIdsMatched;
+};
+
+/**
+ * Complete fileIds with ids from the families that match the data_type
+ *
+ * @param esClient
+ * @param fileIds
+ */
+const getFamilyIds = async (esClient: Client, fileIds: string[]): Promise<string[]> => {
+  const filesInfos = await getFilesInfo(fileIds, esClient);
+  const filesIdsMatched = await getFilesIdsMatched(filesInfos, esClient);
+  const newFileIds = [...new Set([...fileIds, ...filesIdsMatched])];
+
+  return newFileIds;
+};
+
+export default getFamilyIds;

src/services/sets/index.ts

@@ -0,0 +1,164 @@
+import { addSqonToSetSqon, removeSqonToSetSqon } from '@ferlab/next/lib/sqon/manipulateSqon';
+import { resolveSetsInSqon } from '@ferlab/next/lib/sqon/resolveSetInSqon';
+import { searchSqon } from '@ferlab/next/lib/sqon/searchSqon';
+import { CreateUpdateBody, Output } from '@ferlab/next/lib/usersApi';
+import { deleteUserContent, getUserContents, postUserContent, putUserContent } from '@ferlab/next/lib/usersApi';
+import difference from 'lodash/difference';
+import dropRight from 'lodash/dropRight';
+import union from 'lodash/union';
+
+import getFamilyIds from './getFamilyIds';
+import { SetNotFoundError } from './setError';
+import { CreateSetBody, Set, UpdateSetContentBody, UpdateSetTagBody } from './types';
+
+export const SubActionTypes = {
+  RENAME_TAG: 'RENAME_TAG',
+  ADD_IDS: 'ADD_IDS',
+  REMOVE_IDS: 'REMOVE_IDS',
+};
+
+export const getUserSet = async (
+  accessToken: string,
+  userId: string,
+  setId: string,
+  usersApiURL: string
+): Promise<Output> => {
+  const existingSetsFilterById = (await getUserContents(accessToken, usersApiURL)).filter((r) => r.id === setId);
+
+  if (existingSetsFilterById.length !== 1) {
+    throw new SetNotFoundError('Set to update can not be found !');
+  }
+
+  return existingSetsFilterById[0];
+};
+
+export const getSets = async (accessToken: string, usersApiURL: string): Promise<Set[]> => {
+  const userContents = await getUserContents(accessToken, usersApiURL);
+  return userContents.map((set) => mapResultToSet(set));
+};
+
+export const createSet = async (
+  requestBody: CreateSetBody,
+  accessToken: string,
+  userId: string,
+  usersApiURL,
+  esClient,
+  schema,
+  maxSetContentSize: number
+): Promise<Set> => {
+  const { sqon, sort, type, idField, tag, sharedpublicly, is_phantom_manifest, withFamily } = requestBody;
+  const sqonAfterReplace = await resolveSetsInSqon(sqon, userId, accessToken, usersApiURL);
+  const ids = await searchSqon(sqonAfterReplace, type, sort, idField, esClient, schema, maxSetContentSize);
+  const idsWithFamily = withFamily ? await getFamilyIds(esClient, ids) : ids;
+  const truncatedIds = truncateIds(idsWithFamily, maxSetContentSize);
+
+  const payload = {
+    alias: tag,
+    sharedpublicly,
+    is_phantom_manifest,
+    content: { ids: truncatedIds, setType: type, sqon, sort, idField },
+  } as CreateUpdateBody;
+
+  if (!payload.alias || !payload.content.ids) {
+    throw Error(`Set must have ${!payload.alias ? 'a name' : 'no set ids'}`);
+  }
+  const createResult = await postUserContent(accessToken, payload, usersApiURL);
+
+  const setResult: Set = mapResultToSet(createResult);
+  return setResult;
+};
+
+export const updateSetTag = async (
+  requestBody: UpdateSetTagBody,
+  accessToken: string,
+  userId: string,
+  setId: string,
+  usersApiURL: string
+): Promise<Set> => {
+  const setToUpdate = await getUserSet(accessToken, userId, setId, usersApiURL);
+
+  const payload = {
+    alias: requestBody.newTag,
+    sharedpublicly: setToUpdate.sharedpublicly,
+    content: setToUpdate.content,
+  } as CreateUpdateBody;
+
+  const updateResult = await putUserContent(accessToken, payload, setId, usersApiURL);
+
+  const setResult: Set = mapResultToSet(updateResult);
+  return setResult;
+};
+
+export const updateSetContent = async (
+  requestBody: UpdateSetContentBody,
+  accessToken: string,
+  userId: string,
+  setId: string,
+  esClient,
+  schema,
+  usersApiURL: string,
+  maxSetContentSize: number
+): Promise<Set> => {
+  const setToUpdate = await getUserSet(accessToken, userId, setId, usersApiURL);
+
+  const { sqon, ids, setType } = setToUpdate.content;
+
+  const sqonAfterReplace = await resolveSetsInSqon(requestBody.sqon, userId, accessToken, usersApiURL);
+
+  const newSqonIds = await searchSqon(
+    sqonAfterReplace,
+    setToUpdate.content.setType,
+    setToUpdate.content.sort,
+    setToUpdate.content.idField,
+    esClient,
+    schema,
+    maxSetContentSize
+  );
+
+  if (setType !== setToUpdate.content.setType) {
+    throw new Error('Cannot add/remove from a set not of the same type');
+  }
+
+  const existingSqonWithNewSqon =
+    requestBody.subAction === SubActionTypes.ADD_IDS
+      ? addSqonToSetSqon(sqon, requestBody.sqon)
+      : removeSqonToSetSqon(sqon, requestBody.sqon);
+
+  const existingIdsWithNewIds =
+    requestBody.subAction === SubActionTypes.ADD_IDS ? union(ids, newSqonIds) : difference(ids, newSqonIds);
+  const truncatedIds = truncateIds(existingIdsWithNewIds, maxSetContentSize);
+
+  const payload = {
+    alias: setToUpdate.alias,
+    sharedpublicly: setToUpdate.sharedpublicly,
+    content: { ...setToUpdate.content, sqon: existingSqonWithNewSqon, ids: truncatedIds },
+  } as CreateUpdateBody;
+
+  const updateResult = await putUserContent(accessToken, payload, setId, usersApiURL);
+
+  const setResult: Set = mapResultToSet(updateResult);
+  return setResult;
+};
+
+export const deleteSet = async (accessToken: string, setId: string, usersApiURL: string): Promise<boolean> => {
+  const deleteResult = await deleteUserContent(accessToken, setId, usersApiURL);
+  return deleteResult;
+};
+
+const mapResultToSet = (output: Output): Set => ({
+  id: output.id,
+  tag: output.alias,
+  size: output.content.ids.length,
+  updated_date: output.updated_date,
+  setType: output.content.setType,
+  ids: output.content.ids,
+  sharedpublicly: output.sharedpublicly,
+  is_phantom_manifest: output.is_phantom_manifest,
+});
+
+const truncateIds = (ids: string[], maxSetContentSize: number): string[] => {
+  if (ids.length <= maxSetContentSize) {
+    return ids;
+  }
+  return dropRight(ids, ids.length - maxSetContentSize);
+};

src/services/sets/setError.ts

@@ -0,0 +1,7 @@
+export class SetNotFoundError extends Error {
+  constructor(message: string) {
+    super(message);
+    Object.setPrototypeOf(this, SetNotFoundError.prototype);
+    this.name = SetNotFoundError.name;
+  }
+}