diff --git a/src/app.test.ts b/src/app.test.ts index e79c75a..650f449 100644 --- a/src/app.test.ts +++ b/src/app.test.ts @@ -1,16 +1,17 @@ -import { createSet, deleteSet, getSets, SubActionTypes, updateSetContent, updateSetTag } from '@ferlab/next/lib/sets'; -import { Set, UpdateSetContentBody, UpdateSetTagBody } from '@ferlab/next/lib/sets/types'; import { jest } from '@jest/globals'; import { Express } from 'express'; import Keycloak from 'keycloak-connect'; import request from 'supertest'; +import { createSet, deleteSet, getSets, SubActionTypes, updateSetContent, updateSetTag } from '#src/services/sets'; +import { Set, UpdateSetContentBody, UpdateSetTagBody } from '#src/services/sets/types'; + import buildApp from './app'; import { keycloakClient, keycloakRealm, keycloakURL } from './config/env'; import { getStatistics, Statistics } from './endpoints/statistics'; import { getToken, publicKey } from './utils/authTestUtils'; -jest.mock('@ferlab/next/lib/sets/index'); +jest.mock('#src/services/sets/index'); jest.mock('./endpoints/statistics'); //todo: fix tests diff --git a/src/app.ts b/src/app.ts index 2e9ec41..5057935 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,4 +1,3 @@ -import { SetSqon } from '@ferlab/next/lib/sets/types'; import resolveSetIdMiddleware from '@ferlab/next/lib/sqon/resolveSetIdMiddleware'; import compression from 'compression'; import cors from 'cors'; @@ -7,6 +6,8 @@ import { StatusCodes } from 'http-status-codes'; import { Keycloak } from 'keycloak-connect'; import NodeCache from 'node-cache'; +import { SetSqon } from '#src/services/sets/types'; + import packageJson from '../package.json' assert { type: 'json' }; import { cacheTTL, esHost, keycloakURL, usersApiURL } from './config/env'; import { getExtendedMapping } from './endpoints/extendedMapping'; diff --git a/src/endpoints/phenotypes.ts b/src/endpoints/phenotypes.ts index a459cc6..7bd6de6 100644 --- a/src/endpoints/phenotypes.ts +++ b/src/endpoints/phenotypes.ts @@ -1,14 +1,13 @@ -import { SetSqon } from '@ferlab/next/lib/sets/types'; import searchSqon from '@ferlab/next/lib/sqon/searchSqon'; import { replaceSetByIds } from '@ferlab/next/lib/sqon/setSqon'; import get from 'lodash/get'; +import { participantBiospecimenKey, participantFileKey, participantKey } from '#src/config/env'; +import { maxSetContentSize, participantIdKey, usersApiURL } from '#src/config/env'; +import runQuery from '#src/graphql/runQuery'; import schema from '#src/graphql/schema'; import esClient from '#src/services/elasticsearch/client'; - -import { participantBiospecimenKey, participantFileKey, participantKey } from '../config/env'; -import { maxSetContentSize, participantIdKey, usersApiURL } from '../config/env'; -import runQuery from '../graphql/runQuery'; +import { SetSqon } from '#src/services/sets/types'; const getPathToParticipantId = (type: string) => { if (type === 'biospecimen') { diff --git a/src/graphql/runQuery.ts b/src/graphql/runQuery.ts index 0d024b8..214d677 100644 --- a/src/graphql/runQuery.ts +++ b/src/graphql/runQuery.ts @@ -1,8 +1,9 @@ -import { SetSqon, Sort } from '@ferlab/next/lib/sets/types'; import { graphql } from 'graphql'; import { ExecutionResult } from 'graphql/execution/execute'; -import esClient from '../services/elasticsearch/client'; +import esClient from '#src/services/elasticsearch/client'; +import { SetSqon, Sort } from '#src/services/sets/types'; + import schema from './schema'; interface IrunQuery { diff --git a/src/routes/sets/index.ts b/src/routes/sets/index.ts index 81b7ff7..87a442d 100644 --- a/src/routes/sets/index.ts +++ b/src/routes/sets/index.ts @@ -1,10 +1,10 @@ -import { createSet, deleteSet, getSets, SubActionTypes, updateSetContent, updateSetTag } from '@ferlab/next/lib/sets'; -import { CreateSetBody, Set, UpdateSetContentBody, UpdateSetTagBody } from '@ferlab/next/lib/sets/types'; import express from 'express'; import { maxSetContentSize, usersApiURL } from '#src/config/env'; import schema from '#src/graphql/schema'; import esClient from '#src/services/elasticsearch/client'; +import { createSet, deleteSet, getSets, SubActionTypes, updateSetContent, updateSetTag } from '#src/services/sets'; +import { CreateSetBody, Set, UpdateSetContentBody, UpdateSetTagBody } from '#src/services/sets/types'; const router = express.Router(); diff --git a/src/services/elasticsearch/utils.ts b/src/services/elasticsearch/utils.ts index 44d368d..f80d8ad 100644 --- a/src/services/elasticsearch/utils.ts +++ b/src/services/elasticsearch/utils.ts @@ -32,3 +32,26 @@ export const getBody = ({ field, value, path, nested = false }) => { query: { bool: { must } }, }; }; + +/** + * Calls the `search` on the given elasticsearch.Client to get a single page of results. + * @param {Object} esClient - an elasticsearch.Client object. + * @param {String} index - the name of the index (or alias) on which to search. + * @param {Object} query - an object containing the query, + */ +export const executeSearch = async (esClient, index, query) => { + const searchParams = { + index, + body: { + ...query, + size: typeof query.size === 'number' ? query.size : 0, + }, + }; + + try { + return esClient.search(searchParams); + } catch (err) { + console.error(`Error searching ES with params ${JSON.stringify(searchParams)}`, err); + throw err; + } +}; diff --git a/src/services/sets/getFamilyIds.ts b/src/services/sets/getFamilyIds.ts new file mode 100644 index 0000000..70651ab --- /dev/null +++ b/src/services/sets/getFamilyIds.ts @@ -0,0 +1,89 @@ +import { Client } from '@opensearch-project/opensearch'; + +import { esFileIndex, maxSetContentSize } from '#src/config/env'; + +import { executeSearch } from '../elasticsearch/utils'; + +interface IFileInfo { + data_type: string; + family_id: string; +} + +/** Get IFileInfo: files data_types and family_ids */ +const getFilesInfo = async (fileIds: string[], es: Client): Promise => { + const esRequest = { + query: { bool: { must: [{ terms: { file_id: fileIds, boost: 0 } }] } }, + _source: ['file_id', 'data_type', 'participants.family_id'], + sort: [{ data_type: { order: 'asc' } }], + size: maxSetContentSize, + }; + const results = await executeSearch(es, esFileIndex, esRequest); + const hits = results?.body?.hits?.hits || []; + const sources = hits.map((hit) => hit._source); + const filesInfos = []; + sources?.forEach((source) => { + source.participants?.forEach((participant) => { + if ( + participant.family_id && + !filesInfos.find((f) => f.family_id === participant.family_id && f.data_type === source.data_type) + ) { + filesInfos.push({ + data_type: source.data_type, + family_id: participant.family_id || '', + }); + } + }); + }); + return filesInfos; +}; + +/** for each filesInfos iteration, get files from file.participants.family_id and file.data_type */ +const getFilesIdsMatched = async (filesInfos: IFileInfo[], es: Client): Promise => { + const filesIdsMatched = []; + const results = await Promise.all( + filesInfos.map((info) => { + const esRequest = { + query: { + bool: { + must: [ + { terms: { data_type: [info.data_type], boost: 0 } }, + { + nested: { + path: 'participants', + query: { bool: { must: [{ match: { ['participants.family_id']: info.family_id } }] } }, + }, + }, + ], + }, + }, + _source: ['file_id'], + size: maxSetContentSize, + }; + return executeSearch(es, esFileIndex, esRequest); + }) + ); + + for (const res of results) { + const hits = res?.body?.hits?.hits || []; + const sources = hits.map((hit) => hit._source); + filesIdsMatched.push(...sources.map((s) => s.file_id)); + } + + return filesIdsMatched; +}; + +/** + * Complete fileIds with ids from the families that match the data_type + * + * @param esClient + * @param fileIds + */ +const getFamilyIds = async (esClient: Client, fileIds: string[]): Promise => { + const filesInfos = await getFilesInfo(fileIds, esClient); + const filesIdsMatched = await getFilesIdsMatched(filesInfos, esClient); + const newFileIds = [...new Set([...fileIds, ...filesIdsMatched])]; + + return newFileIds; +}; + +export default getFamilyIds; diff --git a/src/services/sets/index.ts b/src/services/sets/index.ts new file mode 100644 index 0000000..e5286cb --- /dev/null +++ b/src/services/sets/index.ts @@ -0,0 +1,164 @@ +import { addSqonToSetSqon, removeSqonToSetSqon } from '@ferlab/next/lib/sqon/manipulateSqon'; +import { resolveSetsInSqon } from '@ferlab/next/lib/sqon/resolveSetInSqon'; +import { searchSqon } from '@ferlab/next/lib/sqon/searchSqon'; +import { CreateUpdateBody, Output } from '@ferlab/next/lib/usersApi'; +import { deleteUserContent, getUserContents, postUserContent, putUserContent } from '@ferlab/next/lib/usersApi'; +import difference from 'lodash/difference'; +import dropRight from 'lodash/dropRight'; +import union from 'lodash/union'; + +import getFamilyIds from './getFamilyIds'; +import { SetNotFoundError } from './setError'; +import { CreateSetBody, Set, UpdateSetContentBody, UpdateSetTagBody } from './types'; + +export const SubActionTypes = { + RENAME_TAG: 'RENAME_TAG', + ADD_IDS: 'ADD_IDS', + REMOVE_IDS: 'REMOVE_IDS', +}; + +export const getUserSet = async ( + accessToken: string, + userId: string, + setId: string, + usersApiURL: string +): Promise => { + const existingSetsFilterById = (await getUserContents(accessToken, usersApiURL)).filter((r) => r.id === setId); + + if (existingSetsFilterById.length !== 1) { + throw new SetNotFoundError('Set to update can not be found !'); + } + + return existingSetsFilterById[0]; +}; + +export const getSets = async (accessToken: string, usersApiURL: string): Promise => { + const userContents = await getUserContents(accessToken, usersApiURL); + return userContents.map((set) => mapResultToSet(set)); +}; + +export const createSet = async ( + requestBody: CreateSetBody, + accessToken: string, + userId: string, + usersApiURL, + esClient, + schema, + maxSetContentSize: number +): Promise => { + const { sqon, sort, type, idField, tag, sharedpublicly, is_phantom_manifest, withFamily } = requestBody; + const sqonAfterReplace = await resolveSetsInSqon(sqon, userId, accessToken, usersApiURL); + const ids = await searchSqon(sqonAfterReplace, type, sort, idField, esClient, schema, maxSetContentSize); + const idsWithFamily = withFamily ? await getFamilyIds(esClient, ids) : ids; + const truncatedIds = truncateIds(idsWithFamily, maxSetContentSize); + + const payload = { + alias: tag, + sharedpublicly, + is_phantom_manifest, + content: { ids: truncatedIds, setType: type, sqon, sort, idField }, + } as CreateUpdateBody; + + if (!payload.alias || !payload.content.ids) { + throw Error(`Set must have ${!payload.alias ? 'a name' : 'no set ids'}`); + } + const createResult = await postUserContent(accessToken, payload, usersApiURL); + + const setResult: Set = mapResultToSet(createResult); + return setResult; +}; + +export const updateSetTag = async ( + requestBody: UpdateSetTagBody, + accessToken: string, + userId: string, + setId: string, + usersApiURL: string +): Promise => { + const setToUpdate = await getUserSet(accessToken, userId, setId, usersApiURL); + + const payload = { + alias: requestBody.newTag, + sharedpublicly: setToUpdate.sharedpublicly, + content: setToUpdate.content, + } as CreateUpdateBody; + + const updateResult = await putUserContent(accessToken, payload, setId, usersApiURL); + + const setResult: Set = mapResultToSet(updateResult); + return setResult; +}; + +export const updateSetContent = async ( + requestBody: UpdateSetContentBody, + accessToken: string, + userId: string, + setId: string, + esClient, + schema, + usersApiURL: string, + maxSetContentSize: number +): Promise => { + const setToUpdate = await getUserSet(accessToken, userId, setId, usersApiURL); + + const { sqon, ids, setType } = setToUpdate.content; + + const sqonAfterReplace = await resolveSetsInSqon(requestBody.sqon, userId, accessToken, usersApiURL); + + const newSqonIds = await searchSqon( + sqonAfterReplace, + setToUpdate.content.setType, + setToUpdate.content.sort, + setToUpdate.content.idField, + esClient, + schema, + maxSetContentSize + ); + + if (setType !== setToUpdate.content.setType) { + throw new Error('Cannot add/remove from a set not of the same type'); + } + + const existingSqonWithNewSqon = + requestBody.subAction === SubActionTypes.ADD_IDS + ? addSqonToSetSqon(sqon, requestBody.sqon) + : removeSqonToSetSqon(sqon, requestBody.sqon); + + const existingIdsWithNewIds = + requestBody.subAction === SubActionTypes.ADD_IDS ? union(ids, newSqonIds) : difference(ids, newSqonIds); + const truncatedIds = truncateIds(existingIdsWithNewIds, maxSetContentSize); + + const payload = { + alias: setToUpdate.alias, + sharedpublicly: setToUpdate.sharedpublicly, + content: { ...setToUpdate.content, sqon: existingSqonWithNewSqon, ids: truncatedIds }, + } as CreateUpdateBody; + + const updateResult = await putUserContent(accessToken, payload, setId, usersApiURL); + + const setResult: Set = mapResultToSet(updateResult); + return setResult; +}; + +export const deleteSet = async (accessToken: string, setId: string, usersApiURL: string): Promise => { + const deleteResult = await deleteUserContent(accessToken, setId, usersApiURL); + return deleteResult; +}; + +const mapResultToSet = (output: Output): Set => ({ + id: output.id, + tag: output.alias, + size: output.content.ids.length, + updated_date: output.updated_date, + setType: output.content.setType, + ids: output.content.ids, + sharedpublicly: output.sharedpublicly, + is_phantom_manifest: output.is_phantom_manifest, +}); + +const truncateIds = (ids: string[], maxSetContentSize: number): string[] => { + if (ids.length <= maxSetContentSize) { + return ids; + } + return dropRight(ids, ids.length - maxSetContentSize); +}; diff --git a/src/services/sets/setError.ts b/src/services/sets/setError.ts new file mode 100644 index 0000000..1d8e446 --- /dev/null +++ b/src/services/sets/setError.ts @@ -0,0 +1,7 @@ +export class SetNotFoundError extends Error { + constructor(message: string) { + super(message); + Object.setPrototypeOf(this, SetNotFoundError.prototype); + this.name = SetNotFoundError.name; + } +} diff --git a/src/services/sets/types.ts b/src/services/sets/types.ts new file mode 100644 index 0000000..7c72681 --- /dev/null +++ b/src/services/sets/types.ts @@ -0,0 +1,45 @@ +export type SetSqon = { + op: string; + content: any; +}; + +export type Sort = { + field: string; + order: string; +}; + +export type CreateSetBody = { + projectId: string; + type: string; + sqon: SetSqon; + idField: string; + sort: Sort[]; + tag: string; + sharedpublicly: boolean; + is_phantom_manifest: boolean; + withFamily?: boolean; +}; + +export type UpdateSetTagBody = { + subAction: string; + sourceType: string; + newTag: string; +}; + +export type UpdateSetContentBody = { + subAction: string; + sourceType: string; + sqon: SetSqon; + projectId: string; +}; + +export type Set = { + id: string; + tag: string; + size: number; + updated_date: Date; + setType: string; + ids: string[]; + sharedpublicly: boolean; + is_phantom_manifest: boolean; +}; diff --git a/src/services/usersApi/index.ts b/src/services/usersApi/index.ts new file mode 100644 index 0000000..d28382c --- /dev/null +++ b/src/services/usersApi/index.ts @@ -0,0 +1,131 @@ +import fetch from 'node-fetch'; + +import { SetSqon, Sort } from '../sets/types'; + +export type CreateUpdateBody = { + alias: string; + content: Content; + sharedpublicly: boolean; +}; + +export type Content = { + setType: string; + ids: string[]; + sqon: SetSqon; + sort: Sort[]; + idField: string; +}; + +export type Output = { + id: string; + uid: string; + content: Content; + alias: string; + sharedpublicly: boolean; + is_phantom_manifest: boolean; + creationDate: Date; + updatedDate: Date; + updated_date: Date; +}; + +export const getUserContents = async (accessToken: string, usersApiURL: string): Promise => { + const uri = `${usersApiURL}/user-sets`; + + const response = await fetch(encodeURI(uri), { + method: 'get', + headers: { + Authorization: accessToken, + 'Content-Type': 'application/json', + }, + }); + + const body = await response.json(); + + if (response.status === 200) { + return body as Output[]; + } + + throw new UsersApiError(response.status, body); +}; + +export const postUserContent = async ( + accessToken: string, + set: CreateUpdateBody, + usersApiURL: string +): Promise => { + const uri = `${usersApiURL}/user-sets`; + + const response = await fetch(encodeURI(uri), { + method: 'post', + headers: { + Authorization: accessToken, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(set), + }); + + const body = await response.json(); + + if (response.status < 300) { + return body as Output; + } + + throw new UsersApiError(response.status, body); +}; + +export const putUserContent = async ( + accessToken: string, + set: CreateUpdateBody, + setId: string, + usersApiURL: string +): Promise => { + const uri = `${usersApiURL}/user-sets/${setId}`; + + const response = await fetch(encodeURI(uri), { + method: 'put', + headers: { + Authorization: accessToken, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(set), + }); + + const body = await response.json(); + + if (response.status < 300) { + return body as Output; + } + + throw new UsersApiError(response.status, body); +}; + +export const deleteUserContent = async (accessToken: string, setId: string, usersApiURL: string): Promise => { + const uri = `${usersApiURL}/user-sets/${setId}`; + + const response = await fetch(encodeURI(uri), { + method: 'delete', + headers: { + Authorization: accessToken, + 'Content-Type': 'application/json', + }, + }); + + if (response.status === 200) { + return true; + } + + throw new UsersApiError(response.status, response.body); +}; + +export class UsersApiError extends Error { + public readonly status: number; + public readonly details: unknown; + + constructor(status: number, details: unknown) { + super(`UsersApi returns status ${status}`); + Object.setPrototypeOf(this, UsersApiError.prototype); + this.name = UsersApiError.name; + this.status = status; + this.details = details; + } +} diff --git a/src/utils/errors.ts b/src/utils/errors.ts index 826c858..0fe83b6 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -1,7 +1,8 @@ -import { SetNotFoundError } from '@ferlab/next/lib/sets/setError'; import { NextFunction, Request, Response } from 'express'; import { getReasonPhrase, StatusCodes } from 'http-status-codes'; +import { SetNotFoundError } from '#src/services/sets/setError'; + export const globalErrorHandler = (err: unknown, _req: Request, res: Response, _next: NextFunction): void => { if (err instanceof SetNotFoundError) { res.status(StatusCodes.NOT_FOUND).json({