From 7e0a00f9daf0449ba5f2f68f54d61dd616aaf524 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 3 May 2024 18:45:58 -0300 Subject: [PATCH 01/55] refactor: move contacts and cases types to HrmTypes module --- hrm-domain/hrm-core/case/caseDataAccess.ts | 25 ++--- hrm-domain/hrm-core/case/caseSection/types.ts | 18 +--- hrm-domain/hrm-core/case/caseService.ts | 66 ++----------- .../hrm-core/contact/contactDataAccess.ts | 18 +--- hrm-domain/hrm-core/contact/contactJson.ts | 24 +---- .../hrm-core/contact/sql/contactInsertSql.ts | 21 +--- .../conversation-media-data-access.ts | 90 +++++------------- .../hrm-core/referral/referral-data-access.ts | 9 +- packages/types/HrmTypes/Case.ts | 95 +++++++++++++++++++ packages/types/HrmTypes/CaseSection.ts | 32 +++++++ packages/types/HrmTypes/Contact.ts | 74 +++++++++++++++ packages/types/HrmTypes/ConversationMedia.ts | 65 +++++++++++++ packages/types/HrmTypes/Referral.ts | 22 +++++ packages/types/HrmTypes/index.ts | 21 ++++ 14 files changed, 358 insertions(+), 222 deletions(-) create mode 100644 packages/types/HrmTypes/Case.ts create mode 100644 packages/types/HrmTypes/CaseSection.ts create mode 100644 packages/types/HrmTypes/Contact.ts create mode 100644 packages/types/HrmTypes/ConversationMedia.ts create mode 100644 packages/types/HrmTypes/Referral.ts create mode 100644 packages/types/HrmTypes/index.ts diff --git a/hrm-domain/hrm-core/case/caseDataAccess.ts b/hrm-domain/hrm-core/case/caseDataAccess.ts index 7e5a34b3e..f390c5cd2 100644 --- a/hrm-domain/hrm-core/case/caseDataAccess.ts +++ b/hrm-domain/hrm-core/case/caseDataAccess.ts @@ -29,29 +29,16 @@ import { Contact } from '../contact/contactDataAccess'; import { DateFilter, OrderByDirectionType } from '../sql'; import { TKConditionsSets } from '../permissions/rulesMap'; import { TwilioUser } from '@tech-matters/twilio-worker-auth'; -import { TwilioUserIdentifier, WorkerSID } from '@tech-matters/types'; +import { TwilioUserIdentifier } from '@tech-matters/types'; +import { + PrecalculatedCasePermissionConditions, + CaseRecordCommon, +} from '@tech-matters/types/HrmTypes'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; import { HrmAccountId } from '@tech-matters/types'; -export type PrecalculatedCasePermissionConditions = { - isCaseContactOwner: boolean; // Does the requesting user own any of the contacts currently connected to the case? -}; - -export type CaseRecordCommon = { - info: any; - helpline: string; - status: string; - twilioWorkerId: WorkerSID; - createdBy: TwilioUserIdentifier; - updatedBy: TwilioUserIdentifier; - accountSid: HrmAccountId; - createdAt: string; - updatedAt: string; - statusUpdatedAt?: string; - statusUpdatedBy?: string; - previousStatus?: string; -}; +export { PrecalculatedCasePermissionConditions, CaseRecordCommon }; // Exported for testing export const VALID_CASE_CREATE_FIELDS: (keyof CaseRecordCommon)[] = [ diff --git a/hrm-domain/hrm-core/case/caseSection/types.ts b/hrm-domain/hrm-core/case/caseSection/types.ts index 7c373821b..0a12dd3db 100644 --- a/hrm-domain/hrm-core/case/caseSection/types.ts +++ b/hrm-domain/hrm-core/case/caseSection/types.ts @@ -13,23 +13,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ +import { CaseSectionRecord, CaseSection } from '@tech-matters/types/HrmTypes'; -import { HrmAccountId } from '@tech-matters/types'; - -export type CaseSectionRecord = { - caseId: number; - sectionType: string; - sectionId: string; - sectionTypeSpecificData: Record; - accountSid: HrmAccountId; - createdAt: string; - createdBy: string; - updatedAt?: string; - updatedBy?: string; - eventTimestamp: string; -}; - -export type CaseSection = Omit; +export { CaseSectionRecord, CaseSection }; export type CaseSectionUpdate = Omit< CaseSection, diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index dfe111f43..f5050542c 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -23,7 +23,6 @@ import { CaseListConfiguration, CaseListFilters, CaseRecord, - CaseRecordCommon, CaseSearchCriteria, SearchQueryFunction, create, @@ -35,77 +34,26 @@ import { updateCaseInfo, } from './caseDataAccess'; import { randomUUID } from 'crypto'; -import type { Contact } from '../contact/contactDataAccess'; import { InitializedCan } from '../permissions/initializeCanForRules'; import type { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { bindApplyTransformations as bindApplyContactTransformations } from '../contact/contactService'; import type { Profile } from '../profile/profileDataAccess'; import type { PaginationQuery } from '../search'; import { HrmAccountId, TResult, newErr, newOk } from '@tech-matters/types'; +import { + WELL_KNOWN_CASE_SECTION_NAMES, + CaseService, + CaseInfoSection, +} from '@tech-matters/types/HrmTypes'; import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; +export { WELL_KNOWN_CASE_SECTION_NAMES, CaseService, CaseInfoSection }; + const CASE_OVERVIEW_PROPERTIES = ['summary', 'followUpDate', 'childIsAtRisk'] as const; type CaseOverviewProperties = (typeof CASE_OVERVIEW_PROPERTIES)[number]; -type CaseSection = Omit; - -type CaseInfoSection = { - id: string; - twilioWorkerId: string; - updatedAt?: string; - updatedBy?: string; -} & Record; - -const getSectionSpecificDataFromNotesOrReferrals = ( - caseSection: CaseInfoSection, -): Record => { - const { - id, - twilioWorkerId, - createdAt, - updatedBy, - updatedAt, - accountSid, - ...sectionSpecificData - } = caseSection; - return sectionSpecificData; -}; - -export const WELL_KNOWN_CASE_SECTION_NAMES = { - households: { getSectionSpecificData: s => s.household, sectionTypeName: 'household' }, - perpetrators: { - getSectionSpecificData: s => s.perpetrator, - sectionTypeName: 'perpetrator', - }, - incidents: { getSectionSpecificData: s => s.incident, sectionTypeName: 'incident' }, - counsellorNotes: { - getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, - sectionTypeName: 'note', - }, - referrals: { - getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, - sectionTypeName: 'referral', - }, - documents: { getSectionSpecificData: s => s.document, sectionTypeName: 'document' }, -} as const; - -type PrecalculatedPermissions = Record<'userOwnsContact', boolean>; - -type CaseSectionsMap = { - [k in (typeof WELL_KNOWN_CASE_SECTION_NAMES)[keyof typeof WELL_KNOWN_CASE_SECTION_NAMES]['sectionTypeName']]?: CaseSection[]; -}; - -export type CaseService = CaseRecordCommon & { - id: number; - childName?: string; - categories: Record; - precalculatedPermissions?: PrecalculatedPermissions; - connectedContacts?: Contact[]; - sections: CaseSectionsMap; -}; - type RecursivePartial = { [P in keyof T]?: RecursivePartial; }; diff --git a/hrm-domain/hrm-core/contact/contactDataAccess.ts b/hrm-domain/hrm-core/contact/contactDataAccess.ts index 98f42c38a..9898b7cb2 100644 --- a/hrm-domain/hrm-core/contact/contactDataAccess.ts +++ b/hrm-domain/hrm-core/contact/contactDataAccess.ts @@ -23,29 +23,17 @@ import { selectSingleContactByTaskId, } from './sql/contact-get-sql'; import { INSERT_CONTACT_SQL, NewContactRecord } from './sql/contactInsertSql'; -import { ContactRawJson, ReferralWithoutContactId } from './contactJson'; +import { ContactRawJson } from './contactJson'; import type { ITask } from 'pg-promise'; import { txIfNotInOne } from '../sql'; -import { ConversationMedia } from '../conversation-media/conversation-media'; import { TOUCH_CASE_SQL } from '../case/sql/caseUpdateSql'; import { TKConditionsSets } from '../permissions/rulesMap'; import { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { TwilioUserIdentifier, HrmAccountId } from '@tech-matters/types'; -export type ExistingContactRecord = { - id: number; - accountSid: HrmAccountId; - createdAt: string; - finalizedAt?: string; - updatedAt?: string; - updatedBy?: TwilioUserIdentifier; -} & Partial; +import { ExistingContactRecord, Contact } from '@tech-matters/types/HrmTypes'; -export type Contact = ExistingContactRecord & { - csamReports: any[]; - referrals?: ReferralWithoutContactId[]; - conversationMedia?: ConversationMedia[]; -}; +export { ExistingContactRecord, Contact }; export type SearchParameters = { helpline?: string; diff --git a/hrm-domain/hrm-core/contact/contactJson.ts b/hrm-domain/hrm-core/contact/contactJson.ts index 38ac75c0c..98364ce1f 100644 --- a/hrm-domain/hrm-core/contact/contactJson.ts +++ b/hrm-domain/hrm-core/contact/contactJson.ts @@ -13,27 +13,5 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { Referral } from '../referral/referral-data-access'; -/** - * This and contained types are copied from Flex - */ -export type ContactRawJson = { - definitionVersion?: string; - callType: string; - childInformation: { - [key: string]: string | boolean; - }; - callerInformation?: { - [key: string]: string | boolean; - }; - categories: Record; - caseInformation: { - [key: string]: string | boolean; - }; - contactlessTask?: { [key: string]: string | boolean }; - referrals?: Referral[]; -}; - -// Represents a referral when part of a contact structure, so no contact ID -export type ReferralWithoutContactId = Omit; +export { ContactRawJson, ReferralWithoutContactId } from '@tech-matters/types/HrmTypes'; diff --git a/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts b/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts index 91c923e49..c9dac3357 100644 --- a/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts +++ b/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts @@ -14,27 +14,10 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { ContactRawJson } from '../contactJson'; import { selectSingleContactByTaskId } from './contact-get-sql'; -import { TwilioUserIdentifier, WorkerSID } from '@tech-matters/types'; +import { NewContactRecord } from '@tech-matters/types/HrmTypes'; -export type NewContactRecord = { - rawJson: ContactRawJson; - queueName: string; - twilioWorkerId?: WorkerSID; - createdBy?: TwilioUserIdentifier; - helpline?: string; - number?: string; - channel?: string; - conversationDuration: number; - timeOfContact?: string; - taskId: string; - channelSid?: string; - serviceSid?: string; - caseId?: string; - profileId?: number; - identifierId?: number; -}; +export { NewContactRecord }; export const INSERT_CONTACT_SQL = ` WITH existing AS ( diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts index 18b7d4c51..7dbd580a9 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts @@ -32,73 +32,33 @@ import { import { insertConversationMediaSql } from './sql/conversation-media-insert-sql'; import { updateSpecificDataByIdSql } from './sql/conversation-media-update-sql'; import { HrmAccountId } from '@tech-matters/types'; +import { + S3ContactMediaType, + S3StoredTranscript, + S3StoredRecording, + S3StoredConversationMedia, + ConversationMedia, + NewConversationMedia, + isTwilioStoredMedia, + isS3StoredTranscript, + isS3StoredTranscriptPending, + isS3StoredRecording, + isS3StoredConversationMedia, +} from '@tech-matters/types/HrmTypes'; -/** - * - */ -type ConversationMediaCommons = { - id: number; - contactId: number; - accountSid: HrmAccountId; - createdAt: Date; - updatedAt: Date; -}; - -export enum S3ContactMediaType { - RECORDING = 'recording', - TRANSCRIPT = 'transcript', -} - -type NewTwilioStoredMedia = { - storeType: 'twilio'; - storeTypeSpecificData: { reservationSid: string }; -}; -type TwilioStoredMedia = ConversationMediaCommons & NewTwilioStoredMedia; - -type NewS3StoredTranscript = { - storeType: 'S3'; - storeTypeSpecificData: { - type: S3ContactMediaType.TRANSCRIPT; - location?: { - bucket: string; - key: string; - }; - }; -}; - -type NewS3StoredRecording = { - storeType: 'S3'; - storeTypeSpecificData: { - type: S3ContactMediaType.RECORDING; - location?: { - bucket: string; - key: string; - }; - }; +export { + S3ContactMediaType, + S3StoredTranscript, + S3StoredRecording, + S3StoredConversationMedia, + ConversationMedia, + NewConversationMedia, + isTwilioStoredMedia, + isS3StoredTranscript, + isS3StoredTranscriptPending, + isS3StoredRecording, + isS3StoredConversationMedia, }; -export type S3StoredTranscript = ConversationMediaCommons & NewS3StoredTranscript; -export type S3StoredRecording = ConversationMediaCommons & NewS3StoredRecording; -export type S3StoredConversationMedia = S3StoredTranscript | S3StoredRecording; - -export type ConversationMedia = TwilioStoredMedia | S3StoredConversationMedia; - -export type NewConversationMedia = - | NewTwilioStoredMedia - | NewS3StoredTranscript - | NewS3StoredRecording; - -export const isTwilioStoredMedia = (m: ConversationMedia): m is TwilioStoredMedia => - m.storeType === 'twilio'; -export const isS3StoredTranscript = (m: ConversationMedia): m is S3StoredTranscript => - // eslint-disable-next-line @typescript-eslint/no-use-before-define - m.storeType === 'S3' && m.storeTypeSpecificData?.type === S3ContactMediaType.TRANSCRIPT; -export const isS3StoredTranscriptPending = (m: ConversationMedia) => - isS3StoredTranscript(m) && !m.storeTypeSpecificData?.location; -export const isS3StoredRecording = (m: ConversationMedia): m is S3StoredRecording => - m.storeType === 'S3' && m.storeTypeSpecificData?.type === S3ContactMediaType.RECORDING; -export const isS3StoredConversationMedia = ( - m: ConversationMedia, -): m is S3StoredConversationMedia => isS3StoredTranscript(m) || isS3StoredRecording(m); export const create = (task?) => diff --git a/hrm-domain/hrm-core/referral/referral-data-access.ts b/hrm-domain/hrm-core/referral/referral-data-access.ts index 66c92cede..36d5189ca 100644 --- a/hrm-domain/hrm-core/referral/referral-data-access.ts +++ b/hrm-domain/hrm-core/referral/referral-data-access.ts @@ -14,6 +14,8 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ +import { Referral } from '@tech-matters/types/HrmTypes'; + import { insertReferralSql } from './sql/referral-insert-sql'; import { DatabaseForeignKeyViolationError, @@ -49,12 +51,7 @@ export class OrphanedReferralError extends Error { } } -export type Referral = { - contactId: string; - resourceId: string; - referredAt: string; - resourceName?: string; -}; +export { Referral }; export const createReferralRecord = (task?) => diff --git a/packages/types/HrmTypes/Case.ts b/packages/types/HrmTypes/Case.ts new file mode 100644 index 000000000..24c0cdb67 --- /dev/null +++ b/packages/types/HrmTypes/Case.ts @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '..'; +import { CaseSectionRecord } from './CaseSection'; +import { Contact } from './Contact'; + +type CaseSection = Omit; + +export type PrecalculatedCasePermissionConditions = { + isCaseContactOwner: boolean; // Does the requesting user own any of the contacts currently connected to the case? +}; + +export type CaseRecordCommon = { + info: any; + helpline: string; + status: string; + twilioWorkerId: WorkerSID; + createdBy: TwilioUserIdentifier; + updatedBy: TwilioUserIdentifier; + accountSid: HrmAccountId; + createdAt: string; + updatedAt: string; + statusUpdatedAt?: string; + statusUpdatedBy?: string; + previousStatus?: string; +}; + +export type CaseInfoSection = { + id: string; + twilioWorkerId: string; + updatedAt?: string; + updatedBy?: string; +} & Record; + +const getSectionSpecificDataFromNotesOrReferrals = ( + caseSection: CaseInfoSection, +): Record => { + const { + id, + twilioWorkerId, + createdAt, + updatedBy, + updatedAt, + accountSid, + ...sectionSpecificData + } = caseSection; + return sectionSpecificData; +}; + +export const WELL_KNOWN_CASE_SECTION_NAMES = { + households: { getSectionSpecificData: s => s.household, sectionTypeName: 'household' }, + perpetrators: { + getSectionSpecificData: s => s.perpetrator, + sectionTypeName: 'perpetrator', + }, + incidents: { getSectionSpecificData: s => s.incident, sectionTypeName: 'incident' }, + counsellorNotes: { + getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, + sectionTypeName: 'note', + }, + referrals: { + getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, + sectionTypeName: 'referral', + }, + documents: { getSectionSpecificData: s => s.document, sectionTypeName: 'document' }, +} as const; + +type PrecalculatedPermissions = Record<'userOwnsContact', boolean>; + +type CaseSectionsMap = { + [k in (typeof WELL_KNOWN_CASE_SECTION_NAMES)[keyof typeof WELL_KNOWN_CASE_SECTION_NAMES]['sectionTypeName']]?: CaseSection[]; +}; + +export type CaseService = CaseRecordCommon & { + id: number; + childName?: string; + categories: Record; + precalculatedPermissions?: PrecalculatedPermissions; + connectedContacts?: Contact[]; + sections: CaseSectionsMap; +}; diff --git a/packages/types/HrmTypes/CaseSection.ts b/packages/types/HrmTypes/CaseSection.ts new file mode 100644 index 000000000..fa9263253 --- /dev/null +++ b/packages/types/HrmTypes/CaseSection.ts @@ -0,0 +1,32 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { HrmAccountId } from '..'; + +export type CaseSectionRecord = { + caseId: number; + sectionType: string; + sectionId: string; + sectionTypeSpecificData: Record; + accountSid: HrmAccountId; + createdAt: string; + createdBy: string; + updatedAt?: string; + updatedBy?: string; + eventTimestamp: string; +}; + +export type CaseSection = Omit; diff --git a/packages/types/HrmTypes/Contact.ts b/packages/types/HrmTypes/Contact.ts new file mode 100644 index 000000000..c02b9fa1f --- /dev/null +++ b/packages/types/HrmTypes/Contact.ts @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ +import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '..'; +import { ConversationMedia } from './ConversationMedia'; +import { Referral } from './Referral'; + +/** + * This and contained types are copied from Flex + */ +export type ContactRawJson = { + definitionVersion?: string; + callType: string; + childInformation: { + [key: string]: string | boolean; + }; + callerInformation?: { + [key: string]: string | boolean; + }; + categories: Record; + caseInformation: { + [key: string]: string | boolean; + }; + contactlessTask?: { [key: string]: string | boolean }; + referrals?: Referral[]; +}; + +// Represents a referral when part of a contact structure, so no contact ID +export type ReferralWithoutContactId = Omit; + +export type NewContactRecord = { + rawJson: ContactRawJson; + queueName: string; + twilioWorkerId?: WorkerSID; + createdBy?: TwilioUserIdentifier; + helpline?: string; + number?: string; + channel?: string; + conversationDuration: number; + timeOfContact?: string; + taskId: string; + channelSid?: string; + serviceSid?: string; + caseId?: string; + profileId?: number; + identifierId?: number; +}; + +export type ExistingContactRecord = { + id: number; + accountSid: HrmAccountId; + createdAt: string; + finalizedAt?: string; + updatedAt?: string; + updatedBy?: TwilioUserIdentifier; +} & Partial; + +export type Contact = ExistingContactRecord & { + csamReports: any[]; + referrals?: ReferralWithoutContactId[]; + conversationMedia?: ConversationMedia[]; +}; diff --git a/packages/types/HrmTypes/ConversationMedia.ts b/packages/types/HrmTypes/ConversationMedia.ts new file mode 100644 index 000000000..17b678248 --- /dev/null +++ b/packages/types/HrmTypes/ConversationMedia.ts @@ -0,0 +1,65 @@ +import { HrmAccountId } from '../HrmAccountId'; + +type ConversationMediaCommons = { + id: number; + contactId: number; + accountSid: HrmAccountId; + createdAt: Date; + updatedAt: Date; +}; + +export enum S3ContactMediaType { + RECORDING = 'recording', + TRANSCRIPT = 'transcript', +} + +type NewTwilioStoredMedia = { + storeType: 'twilio'; + storeTypeSpecificData: { reservationSid: string }; +}; +type TwilioStoredMedia = ConversationMediaCommons & NewTwilioStoredMedia; + +type NewS3StoredTranscript = { + storeType: 'S3'; + storeTypeSpecificData: { + type: S3ContactMediaType.TRANSCRIPT; + location?: { + bucket: string; + key: string; + }; + }; +}; + +type NewS3StoredRecording = { + storeType: 'S3'; + storeTypeSpecificData: { + type: S3ContactMediaType.RECORDING; + location?: { + bucket: string; + key: string; + }; + }; +}; +export type S3StoredTranscript = ConversationMediaCommons & NewS3StoredTranscript; +export type S3StoredRecording = ConversationMediaCommons & NewS3StoredRecording; +export type S3StoredConversationMedia = S3StoredTranscript | S3StoredRecording; + +export type ConversationMedia = TwilioStoredMedia | S3StoredConversationMedia; + +export type NewConversationMedia = + | NewTwilioStoredMedia + | NewS3StoredTranscript + | NewS3StoredRecording; + +export const isTwilioStoredMedia = (m: ConversationMedia): m is TwilioStoredMedia => + m.storeType === 'twilio'; +export const isS3StoredTranscript = (m: ConversationMedia): m is S3StoredTranscript => + // eslint-disable-next-line @typescript-eslint/no-use-before-define + m.storeType === 'S3' && m.storeTypeSpecificData?.type === S3ContactMediaType.TRANSCRIPT; +export const isS3StoredTranscriptPending = (m: ConversationMedia) => + isS3StoredTranscript(m) && !m.storeTypeSpecificData?.location; +export const isS3StoredRecording = (m: ConversationMedia): m is S3StoredRecording => + m.storeType === 'S3' && m.storeTypeSpecificData?.type === S3ContactMediaType.RECORDING; +export const isS3StoredConversationMedia = ( + m: ConversationMedia, +): m is S3StoredConversationMedia => isS3StoredTranscript(m) || isS3StoredRecording(m); diff --git a/packages/types/HrmTypes/Referral.ts b/packages/types/HrmTypes/Referral.ts new file mode 100644 index 000000000..4b79eee96 --- /dev/null +++ b/packages/types/HrmTypes/Referral.ts @@ -0,0 +1,22 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export type Referral = { + contactId: string; + resourceId: string; + referredAt: string; + resourceName?: string; +}; diff --git a/packages/types/HrmTypes/index.ts b/packages/types/HrmTypes/index.ts new file mode 100644 index 000000000..8366a362a --- /dev/null +++ b/packages/types/HrmTypes/index.ts @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export * from './Contact'; +export * from './Referral'; +export * from './ConversationMedia'; +export * from './Case'; +export * from './CaseSection'; From 0c62562703a9b094851878d9e2aab240df31b8c4 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 3 May 2024 18:52:54 -0300 Subject: [PATCH 02/55] chore: license header --- packages/types/HrmTypes/ConversationMedia.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/types/HrmTypes/ConversationMedia.ts b/packages/types/HrmTypes/ConversationMedia.ts index 17b678248..a519cf178 100644 --- a/packages/types/HrmTypes/ConversationMedia.ts +++ b/packages/types/HrmTypes/ConversationMedia.ts @@ -1,3 +1,19 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + import { HrmAccountId } from '../HrmAccountId'; type ConversationMediaCommons = { From 03d3a216c176a264ec7498805e944a0dc183c14c Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 6 May 2024 15:36:14 -0300 Subject: [PATCH 03/55] chore: moved hrm types to own package hrm-types --- hrm-domain/hrm-core/case/caseDataAccess.ts | 2 +- hrm-domain/hrm-core/case/caseSection/types.ts | 2 +- hrm-domain/hrm-core/case/caseService.ts | 2 +- .../hrm-core/contact/contactDataAccess.ts | 2 +- hrm-domain/hrm-core/contact/contactJson.ts | 2 +- .../hrm-core/contact/sql/contactInsertSql.ts | 2 +- .../conversation-media-data-access.ts | 2 +- hrm-domain/hrm-core/package.json | 1 + .../hrm-core/referral/referral-data-access.ts | 2 +- hrm-domain/hrm-service/tsconfig.build.json | 1 + .../packages/hrm-search-config/package.json | 1 + .../packages/hrm-types}/Case.ts | 2 +- .../packages/hrm-types}/CaseSection.ts | 2 +- .../packages/hrm-types}/Contact.ts | 2 +- .../packages/hrm-types}/ConversationMedia.ts | 2 +- .../packages/hrm-types}/Referral.ts | 0 .../packages/hrm-types}/index.ts | 0 hrm-domain/packages/hrm-types/package.json | 15 +++++++++++ hrm-domain/packages/hrm-types/tsconfig.json | 8 ++++++ package-lock.json | 27 +++++++++++++++++++ tsconfig.json | 1 + 21 files changed, 66 insertions(+), 12 deletions(-) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/Case.ts (97%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/CaseSection.ts (95%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/Contact.ts (96%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/ConversationMedia.ts (98%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/Referral.ts (100%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/index.ts (100%) create mode 100644 hrm-domain/packages/hrm-types/package.json create mode 100644 hrm-domain/packages/hrm-types/tsconfig.json diff --git a/hrm-domain/hrm-core/case/caseDataAccess.ts b/hrm-domain/hrm-core/case/caseDataAccess.ts index f390c5cd2..376d70aa3 100644 --- a/hrm-domain/hrm-core/case/caseDataAccess.ts +++ b/hrm-domain/hrm-core/case/caseDataAccess.ts @@ -33,7 +33,7 @@ import { TwilioUserIdentifier } from '@tech-matters/types'; import { PrecalculatedCasePermissionConditions, CaseRecordCommon, -} from '@tech-matters/types/HrmTypes'; +} from '@tech-matters/hrm-types'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; import { HrmAccountId } from '@tech-matters/types'; diff --git a/hrm-domain/hrm-core/case/caseSection/types.ts b/hrm-domain/hrm-core/case/caseSection/types.ts index 0a12dd3db..471f0ae6b 100644 --- a/hrm-domain/hrm-core/case/caseSection/types.ts +++ b/hrm-domain/hrm-core/case/caseSection/types.ts @@ -13,7 +13,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { CaseSectionRecord, CaseSection } from '@tech-matters/types/HrmTypes'; +import { CaseSectionRecord, CaseSection } from '@tech-matters/hrm-types'; export { CaseSectionRecord, CaseSection }; diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index f5050542c..7dce4a355 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -44,7 +44,7 @@ import { WELL_KNOWN_CASE_SECTION_NAMES, CaseService, CaseInfoSection, -} from '@tech-matters/types/HrmTypes'; +} from '@tech-matters/hrm-types'; import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; diff --git a/hrm-domain/hrm-core/contact/contactDataAccess.ts b/hrm-domain/hrm-core/contact/contactDataAccess.ts index 9898b7cb2..f301c048a 100644 --- a/hrm-domain/hrm-core/contact/contactDataAccess.ts +++ b/hrm-domain/hrm-core/contact/contactDataAccess.ts @@ -31,7 +31,7 @@ import { TKConditionsSets } from '../permissions/rulesMap'; import { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { TwilioUserIdentifier, HrmAccountId } from '@tech-matters/types'; -import { ExistingContactRecord, Contact } from '@tech-matters/types/HrmTypes'; +import { ExistingContactRecord, Contact } from '@tech-matters/hrm-types'; export { ExistingContactRecord, Contact }; diff --git a/hrm-domain/hrm-core/contact/contactJson.ts b/hrm-domain/hrm-core/contact/contactJson.ts index 98364ce1f..4cdf2d8c9 100644 --- a/hrm-domain/hrm-core/contact/contactJson.ts +++ b/hrm-domain/hrm-core/contact/contactJson.ts @@ -14,4 +14,4 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -export { ContactRawJson, ReferralWithoutContactId } from '@tech-matters/types/HrmTypes'; +export { ContactRawJson, ReferralWithoutContactId } from '@tech-matters/hrm-types'; diff --git a/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts b/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts index c9dac3357..bb0cc49cf 100644 --- a/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts +++ b/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts @@ -15,7 +15,7 @@ */ import { selectSingleContactByTaskId } from './contact-get-sql'; -import { NewContactRecord } from '@tech-matters/types/HrmTypes'; +import { NewContactRecord } from '@tech-matters/hrm-types'; export { NewContactRecord }; diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts index 7dbd580a9..c02a7eeee 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts @@ -44,7 +44,7 @@ import { isS3StoredTranscriptPending, isS3StoredRecording, isS3StoredConversationMedia, -} from '@tech-matters/types/HrmTypes'; +} from '@tech-matters/hrm-types'; export { S3ContactMediaType, diff --git a/hrm-domain/hrm-core/package.json b/hrm-domain/hrm-core/package.json index f70df87e5..efe0e6757 100644 --- a/hrm-domain/hrm-core/package.json +++ b/hrm-domain/hrm-core/package.json @@ -23,6 +23,7 @@ }, "homepage": "https://github.com/tech-matters/hrm#readme", "dependencies": { + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", diff --git a/hrm-domain/hrm-core/referral/referral-data-access.ts b/hrm-domain/hrm-core/referral/referral-data-access.ts index 36d5189ca..677f6d4ea 100644 --- a/hrm-domain/hrm-core/referral/referral-data-access.ts +++ b/hrm-domain/hrm-core/referral/referral-data-access.ts @@ -14,7 +14,7 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { Referral } from '@tech-matters/types/HrmTypes'; +import { Referral } from '@tech-matters/hrm-types'; import { insertReferralSql } from './sql/referral-insert-sql'; import { diff --git a/hrm-domain/hrm-service/tsconfig.build.json b/hrm-domain/hrm-service/tsconfig.build.json index 6287a8075..7694eb726 100644 --- a/hrm-domain/hrm-service/tsconfig.build.json +++ b/hrm-domain/hrm-service/tsconfig.build.json @@ -15,6 +15,7 @@ { "path": "packages/service-discovery" }, { "path": "resources-domain/packages/resources-search-config" }, { "path": "resources-domain/resources-service" }, + { "path": "hrm-domain/packages/hrm-types" }, { "path": "hrm-domain/packages/hrm-search-config" }, { "path": "hrm-domain/hrm-core" }, { "path": "hrm-domain/scheduled-tasks/hrm-data-pull" }, diff --git a/hrm-domain/packages/hrm-search-config/package.json b/hrm-domain/packages/hrm-search-config/package.json index 00cedf47a..1e43204bd 100644 --- a/hrm-domain/packages/hrm-search-config/package.json +++ b/hrm-domain/packages/hrm-search-config/package.json @@ -7,6 +7,7 @@ "license": "AGPL", "dependencies": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/types": "^1.0.0" }, "devDependencies": { diff --git a/packages/types/HrmTypes/Case.ts b/hrm-domain/packages/hrm-types/Case.ts similarity index 97% rename from packages/types/HrmTypes/Case.ts rename to hrm-domain/packages/hrm-types/Case.ts index 24c0cdb67..7d07b5524 100644 --- a/packages/types/HrmTypes/Case.ts +++ b/hrm-domain/packages/hrm-types/Case.ts @@ -14,7 +14,7 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '..'; +import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '@tech-matters/types'; import { CaseSectionRecord } from './CaseSection'; import { Contact } from './Contact'; diff --git a/packages/types/HrmTypes/CaseSection.ts b/hrm-domain/packages/hrm-types/CaseSection.ts similarity index 95% rename from packages/types/HrmTypes/CaseSection.ts rename to hrm-domain/packages/hrm-types/CaseSection.ts index fa9263253..76bffac48 100644 --- a/packages/types/HrmTypes/CaseSection.ts +++ b/hrm-domain/packages/hrm-types/CaseSection.ts @@ -14,7 +14,7 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { HrmAccountId } from '..'; +import { HrmAccountId } from '@tech-matters/types'; export type CaseSectionRecord = { caseId: number; diff --git a/packages/types/HrmTypes/Contact.ts b/hrm-domain/packages/hrm-types/Contact.ts similarity index 96% rename from packages/types/HrmTypes/Contact.ts rename to hrm-domain/packages/hrm-types/Contact.ts index c02b9fa1f..c1bf79121 100644 --- a/packages/types/HrmTypes/Contact.ts +++ b/hrm-domain/packages/hrm-types/Contact.ts @@ -13,7 +13,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '..'; +import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '@tech-matters/types'; import { ConversationMedia } from './ConversationMedia'; import { Referral } from './Referral'; diff --git a/packages/types/HrmTypes/ConversationMedia.ts b/hrm-domain/packages/hrm-types/ConversationMedia.ts similarity index 98% rename from packages/types/HrmTypes/ConversationMedia.ts rename to hrm-domain/packages/hrm-types/ConversationMedia.ts index a519cf178..75e3bb94f 100644 --- a/packages/types/HrmTypes/ConversationMedia.ts +++ b/hrm-domain/packages/hrm-types/ConversationMedia.ts @@ -14,7 +14,7 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { HrmAccountId } from '../HrmAccountId'; +import { HrmAccountId } from '@tech-matters/types'; type ConversationMediaCommons = { id: number; diff --git a/packages/types/HrmTypes/Referral.ts b/hrm-domain/packages/hrm-types/Referral.ts similarity index 100% rename from packages/types/HrmTypes/Referral.ts rename to hrm-domain/packages/hrm-types/Referral.ts diff --git a/packages/types/HrmTypes/index.ts b/hrm-domain/packages/hrm-types/index.ts similarity index 100% rename from packages/types/HrmTypes/index.ts rename to hrm-domain/packages/hrm-types/index.ts diff --git a/hrm-domain/packages/hrm-types/package.json b/hrm-domain/packages/hrm-types/package.json new file mode 100644 index 000000000..d141809d7 --- /dev/null +++ b/hrm-domain/packages/hrm-types/package.json @@ -0,0 +1,15 @@ +{ + "name": "@tech-matters/hrm-types", + "version": "1.0.0", + "description": "", + "main": "dist/index.js", + "author": "", + "license": "AGPL", + "dependencies": { + "@tech-matters/types": "^1.0.0" + }, + "devDependencies": { + "@tsconfig/node16": "^1.0.3" + }, + "scripts": {} +} diff --git a/hrm-domain/packages/hrm-types/tsconfig.json b/hrm-domain/packages/hrm-types/tsconfig.json new file mode 100644 index 000000000..1e9446c0c --- /dev/null +++ b/hrm-domain/packages/hrm-types/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "include": ["*.ts"], + "compilerOptions": { + "outDir": "./dist" + } +} + diff --git a/package-lock.json b/package-lock.json index a0e32ac3b..983d627fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "version": "1.0.0", "license": "AGPL", "dependencies": { + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", @@ -252,6 +253,7 @@ } }, "hrm-domain/lambdas/search-index-consumer": { + "name": "@tech-matters/hrm-search-index-consumer", "version": "1.0.0", "license": "AGPL", "dependencies": { @@ -272,6 +274,7 @@ "license": "AGPL", "dependencies": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/types": "^1.0.0" }, "devDependencies": { @@ -279,6 +282,17 @@ "@tsconfig/node16": "^1.0.3" } }, + "hrm-domain/packages/hrm-types": { + "name": "@tech-matters/hrm-types", + "version": "1.0.0", + "license": "AGPL", + "dependencies": { + "@tech-matters/types": "^1.0.0" + }, + "devDependencies": { + "@tsconfig/node16": "^1.0.3" + } + }, "hrm-domain/scheduled-tasks/case-status-transition": { "name": "@tech-matters/case-status-transition", "version": "1.0.0", @@ -3904,6 +3918,10 @@ "resolved": "hrm-domain/hrm-service", "link": true }, + "node_modules/@tech-matters/hrm-types": { + "resolved": "hrm-domain/packages/hrm-types", + "link": true + }, "node_modules/@tech-matters/http": { "resolved": "packages/http", "link": true @@ -17452,6 +17470,7 @@ "@tech-matters/hrm-core": { "version": "file:hrm-domain/hrm-core", "requires": { + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", @@ -17595,6 +17614,7 @@ "requires": { "@elastic/elasticsearch": "^8.13.1", "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/types": "^1.0.0", "@tsconfig/node16": "^1.0.3" } @@ -17670,6 +17690,13 @@ } } }, + "@tech-matters/hrm-types": { + "version": "file:hrm-domain/packages/hrm-types", + "requires": { + "@tech-matters/types": "^1.0.0", + "@tsconfig/node16": "^1.0.3" + } + }, "@tech-matters/http": { "version": "file:packages/http", "requires": { diff --git a/tsconfig.json b/tsconfig.json index 621b49d05..804066797 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,6 +17,7 @@ { "path": "packages/service-discovery" }, { "path": "lambdas/packages/hrm-authentication" }, { "path": "lambdas/job-complete" }, + { "path": "hrm-domain/packages/hrm-types" }, { "path": "hrm-domain/packages/hrm-search-config" }, { "path": "hrm-domain/lambdas/contact-complete" }, { "path": "hrm-domain/lambdas/contact-retrieve-transcript" }, From 02e996fde5c94eee336fbafb781273cebad5f343 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 6 May 2024 16:48:50 -0300 Subject: [PATCH 04/55] chore: pull hrm-types from correct package --- .../hrm-search-config/convertToIndexDocument.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 51799b58c..d735cf3fd 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -24,10 +24,7 @@ */ import { assertExhaustive } from '@tech-matters/types'; -import type { Contact } from '../../hrm-core/contact/contactService'; -import type { CaseService } from '../../hrm-core/case/caseService'; -// import type { Contact } from '@tech-matters/hrm-core/contact/contactService'; -// import type { CaseService } from '@tech-matters/hrm-core/case/caseService'; +import type { CaseService, Contact } from '@tech-matters/hrm-types'; import type { ContactDocument, CaseDocument, @@ -35,19 +32,24 @@ import type { } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; -type IndexPayloadContact = { +export type IndexContactMessage = { type: 'contact'; contact: Contact; - transcript?: string; }; -type IndexPayloadCase = { +export type IndexCaseMessage = { type: 'case'; case: Omit & { sections: NonNullable; }; }; +export type IndexPayloadContact = IndexContactMessage & { + transcript: NonNullable; +}; + +export type IndexPayloadCase = IndexCaseMessage; + export type IndexPayload = IndexPayloadContact | IndexPayloadCase; const convertToContactDocument = ({ From 9e44933e2d63f1d75404a73ee24588988826965e Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 6 May 2024 16:49:20 -0300 Subject: [PATCH 05/55] chore: added dependencies to search-index-consumer --- hrm-domain/lambdas/search-index-consumer/package.json | 2 ++ hrm-domain/lambdas/search-index-consumer/tsconfig.build.json | 3 ++- package-lock.json | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hrm-domain/lambdas/search-index-consumer/package.json b/hrm-domain/lambdas/search-index-consumer/package.json index de21af940..32608150a 100644 --- a/hrm-domain/lambdas/search-index-consumer/package.json +++ b/hrm-domain/lambdas/search-index-consumer/package.json @@ -7,6 +7,8 @@ "license": "AGPL", "dependencies": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-search-config": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", diff --git a/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json b/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json index 8254c0f0c..a8f8b404b 100644 --- a/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json +++ b/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json @@ -7,7 +7,8 @@ { "path": "packages/job-errors" }, { "path": "packages/ssm-cache" }, { "path": "packages/elasticsearch-client" }, - // { "path": "hrm-domain/packages/hrm-search-config" }, + { "path": "hrm-domain/packages/hrm-types" }, + { "path": "hrm-domain/packages/hrm-search-config" }, { "path": "hrm-domain/lambdas/search-index-consumer" } ] } diff --git a/package-lock.json b/package-lock.json index 983d627fa..19c4b7e32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -258,6 +258,8 @@ "license": "AGPL", "dependencies": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-search-config": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", @@ -17623,6 +17625,8 @@ "version": "file:hrm-domain/lambdas/search-index-consumer", "requires": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-search-config": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", From b3a140c808588ad01fda5129a6bd5e260d43cddf Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 7 May 2024 17:43:41 -0300 Subject: [PATCH 06/55] chore: move types around to ease lambda implementation --- .../convertToIndexDocument.ts | 46 +++++++++++++++---- .../packages/hrm-search-config/index.ts | 6 ++- package-lock.json | 2 + packages/job-errors/index.ts | 7 +++ 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index d735cf3fd..42d9cdbca 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,7 +23,7 @@ * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html */ -import { assertExhaustive } from '@tech-matters/types'; +import { assertExhaustive, AccountSID } from '@tech-matters/types'; import type { CaseService, Contact } from '@tech-matters/hrm-types'; import type { ContactDocument, @@ -32,27 +32,54 @@ import type { } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; -export type IndexContactMessage = { +type IndexContactMessage = { type: 'contact'; contact: Contact; }; -export type IndexCaseMessage = { +type IndexCaseMessage = { type: 'case'; case: Omit & { sections: NonNullable; }; }; -export type IndexPayloadContact = IndexContactMessage & { +export type IndexMessage = { accountSid: AccountSID } & ( + | IndexContactMessage + | IndexCaseMessage +); + +const getContactDocumentId = ({ contact, type }: IndexContactMessage) => + `${type}_${contact.id}`; + +const getCaseDocumentId = ({ case: caseObj, type }: IndexCaseMessage) => + `${type}_${caseObj.id}`; + +export const getDocumentId = (m: IndexMessage) => { + const { type } = m; + switch (type) { + case 'contact': { + return getContactDocumentId(m); + } + case 'case': { + return getCaseDocumentId(m); + } + default: { + return assertExhaustive(type); + } + } +}; + +type IndexPayloadContact = IndexContactMessage & { transcript: NonNullable; }; -export type IndexPayloadCase = IndexCaseMessage; +type IndexPayloadCase = IndexCaseMessage; export type IndexPayload = IndexPayloadContact | IndexPayloadCase; const convertToContactDocument = ({ + type, contact, transcript, }: IndexPayloadContact): CreateIndexConvertedDocument => { @@ -72,8 +99,7 @@ const convertToContactDocument = ({ twilioWorkerId, rawJson, } = contact; - const type = 'contact' as const; - const compundId = `${type}_${id}`; + const compundId = getContactDocumentId({ type, contact }); return { type, @@ -99,6 +125,7 @@ const convertToContactDocument = ({ }; const convertToCaseDocument = ({ + type, case: caseObj, }: IndexPayloadCase): CreateIndexConvertedDocument => { const { @@ -117,8 +144,7 @@ const convertToCaseDocument = ({ sections, info, } = caseObj; - const type = 'case' as const; - const compundId = `${type}_${id}`; + const compundId = getCaseDocumentId({ type, case: caseObj }); const mappedSections: CaseDocument['sections'] = Object.entries(sections).flatMap( ([sectionType, sectionsArray]) => @@ -172,7 +198,7 @@ export const convertToIndexDocument = ( return convertToCaseDocument(payload); } default: { - assertExhaustive(type); + return assertExhaustive(type); } } }; diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index aaa59c323..b8a829737 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -21,7 +21,11 @@ import type { SearchConfiguration, } from '@tech-matters/elasticsearch-client'; -export { HRM_CASES_CONTACTS_INDEX_TYPE } from './hrmIndexDocumentMappings'; +export { + HRM_CASES_CONTACTS_INDEX_TYPE, + CasesContactsDocument, +} from './hrmIndexDocumentMappings'; +export { IndexMessage, IndexPayload, getDocumentId } from './convertToIndexDocument'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { diff --git a/package-lock.json b/package-lock.json index 19c4b7e32..1d3847b6b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -261,6 +261,7 @@ "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", + "@tech-matters/s3-client": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", "@tech-matters/types": "^1.0.0" @@ -17628,6 +17629,7 @@ "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", + "@tech-matters/s3-client": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", "@tech-matters/types": "^1.0.0", diff --git a/packages/job-errors/index.ts b/packages/job-errors/index.ts index fbba9a98e..826a1604d 100644 --- a/packages/job-errors/index.ts +++ b/packages/job-errors/index.ts @@ -21,6 +21,13 @@ export class ContactJobProcessorError extends Error { } } +export class HrmIndexProcessorError extends Error { + constructor(message: string) { + super(message); + this.name = 'HrmIndexProcessorError'; + } +} + export class ResourceImportProcessorError extends Error { constructor(message: string) { super(message); From e3d37cac3f5a988c5edcf907516709507e0ff96d Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 7 May 2024 17:44:20 -0300 Subject: [PATCH 07/55] chore: implement search-index-consumer indexing --- .../lambdas/search-index-consumer/index.ts | 247 +++++++++++++++--- .../search-index-consumer/package.json | 1 + 2 files changed, 217 insertions(+), 31 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index ee9423c78..c635ce720 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -14,47 +14,232 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ import type { SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda'; +import { getClient } from '@tech-matters/elasticsearch-client'; +// import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; +import { HrmIndexProcessorError } from '@tech-matters/job-errors'; +import { + IndexMessage, + HRM_CASES_CONTACTS_INDEX_TYPE, + hrmIndexConfiguration, + IndexPayload, + getDocumentId, +} from '@tech-matters/hrm-search-config'; +import { + AccountSID, + assertExhaustive, + isErr, + newErr, + newOkFromData, +} from '@tech-matters/types'; -export const handler = async (event: SQSEvent): Promise => { - const response: SQSBatchResponse = { batchItemFailures: [] }; - console.debug('Received event:', JSON.stringify(event, null, 2)); - // We need to keep track of the documentId to messageId mapping so we can - // return the correct messageId in the batchItemFailures array on error. - const documentIdToMessageId: Record = {}; +export type MessagesByAccountSid = Record< + AccountSID, + { message: IndexMessage; documentId: string; messageId: string }[] +>; +type PayloadWithMeta = { + payload: IndexPayload; + documentId: string; + messageId: string; +}; +type PayloadsByIndex = { + [indexType: string]: PayloadWithMeta[]; +}; +export type PayloadsByAccountSid = Record; - // Passthrough function to add the documentId to the messageId mapping. - const addDocumentIdToMessageId = (documentId: string, messageId: string) => { - documentIdToMessageId[documentId] = messageId; +type MessagesByDocumentId = { + [documentId: string]: { + documentId: string; + messageId: string; + message: IndexMessage; }; +}; - // Passthrough function to add the documentId to the batchItemFailures array. - // const addDocumentIdToFailures = (documentId: string) => - // response.batchItemFailures.push({ - // itemIdentifier: documentIdToMessageId[documentId], - // }); +const reduceByDocumentId = ( + accum: MessagesByDocumentId, + curr: SQSRecord, +): MessagesByDocumentId => { + const { messageId, body } = curr; + const message = JSON.parse(body) as IndexMessage; - try { - // Map the messages and add the documentId to messageId mapping. - // const messages = mapMessages(event.Records, addDocumentIdToMessageId); - const messages = event.Records.map((record: SQSRecord) => { - const { messageId, body } = record; - const message = JSON.parse(body); - addDocumentIdToMessageId(message.document.id, messageId); + const documentId = getDocumentId(message); - return message; - }); - console.debug('Mapped messages:', JSON.stringify(messages, null, 2)); + return { ...accum, [documentId]: { documentId, messageId, message } }; +}; - console.debug(`Successfully indexed documents`); - } catch (err) { - console.error(new Error('Failed to process search index request'), err); +const groupMessagesByAccountSid = ( + accum: MessagesByAccountSid, + curr: { + documentId: string; + messageId: string; + message: IndexMessage; + }, +): MessagesByAccountSid => { + const { message } = curr; + const { accountSid } = message; - response.batchItemFailures = event.Records.map(record => { + if (!accum[accountSid]) { + return { ...accum, [accountSid]: [curr] }; + } + + return { ...accum, [accountSid]: [...accum[accountSid], curr] }; +}; + +const messagesToPayloadsByIndex = ( + accum: PayloadsByIndex, + currM: { + documentId: string; + messageId: string; + message: IndexMessage; + }, +): PayloadsByIndex => { + const { message } = currM; + + const { type } = message; + + switch (type) { + case 'contact': { + // TODO: Pull the transcripts from S3 (if any) return { - itemIdentifier: record.messageId, + ...accum, + [HRM_CASES_CONTACTS_INDEX_TYPE]: [ + ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), + { ...currM, payload: { ...message, transcript: '' } }, + ], }; - }); + } + case 'case': { + return { + ...accum, + [HRM_CASES_CONTACTS_INDEX_TYPE]: [ + ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), + { ...currM, payload: { ...message } }, + ], + }; + } + default: { + return assertExhaustive(type); + } } +}; + +const indexDocumentsByIndex = + (accountSid: string) => + async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { + // get the client for the accountSid-indexType pair + const client = (await getClient({ accountSid, indexType })).indexClient( + hrmIndexConfiguration, + ); + + const indexed = await Promise.all( + payloads.map(({ documentId, messageId, payload }) => + client + .indexDocument({ + id: documentId, + document: payload, + autocreate: true, + }) + .then(result => ({ + accountSid, + indexType, + documentId, + messageId, + result: newOkFromData(result), + })) + .catch(err => { + console.error( + new HrmIndexProcessorError('Failed to process search index request'), + err, + ); + + return { + accountSid, + indexType, + documentId, + messageId, + result: newErr({ + error: 'ErrorFailedToInex', + message: err instanceof Error ? err.message : String(err), + }), + }; + }), + ), + ); + + return indexed; + }; - return response; +const indexDocumentsByAccount = async ([accountSid, payloadsByIndex]: [ + string, + PayloadsByIndex, +]) => { + const resultsByIndex = await Promise.all( + Object.entries(payloadsByIndex).map(indexDocumentsByIndex(accountSid)), + ); + + return resultsByIndex; +}; + +export const handler = async (event: SQSEvent): Promise => { + console.debug('Received event:', JSON.stringify(event, null, 2)); + + try { + // link each composite "documentId" to it's corresponding "messageId" + const documentIdToMessage = event.Records.reduce(reduceByDocumentId, {}); + + // group the messages by accountSid + const messagesByAccoundSid = Object.values( + documentIdToMessage, + ).reduce(groupMessagesByAccountSid, {}); + + // generate corresponding IndexPayload for each IndexMessage and group them by target indexType + const documentsByAccountSid: PayloadsByAccountSid = Object.fromEntries( + Object.entries(messagesByAccoundSid).map(([accountSid, messages]) => { + const payloads = messages.reduce(messagesToPayloadsByIndex, {}); + + return [accountSid, payloads] as const; + }), + ); + + console.debug('Mapped messages:', JSON.stringify(documentsByAccountSid, null, 2)); + + const resultsByAccount = await Promise.all( + Object.entries(documentsByAccountSid).map(indexDocumentsByAccount), + ); + + console.debug(`Successfully indexed documents`); + + const documentsWithErrors = resultsByAccount + .flat(2) + .filter(({ result }) => isErr(result)); + + if (documentsWithErrors.length) { + console.debug( + 'Errors indexing documents', + JSON.stringify(documentsWithErrors, null, 2), + ); + } + + const response: SQSBatchResponse = { + batchItemFailures: documentsWithErrors.map(({ messageId }) => ({ + itemIdentifier: messageId, + })), + }; + + return response; + } catch (err) { + console.error( + new HrmIndexProcessorError('Failed to process search index request'), + err, + ); + + const response: SQSBatchResponse = { + batchItemFailures: event.Records.map(record => { + return { + itemIdentifier: record.messageId, + }; + }), + }; + + return response; + } }; diff --git a/hrm-domain/lambdas/search-index-consumer/package.json b/hrm-domain/lambdas/search-index-consumer/package.json index 32608150a..77873183a 100644 --- a/hrm-domain/lambdas/search-index-consumer/package.json +++ b/hrm-domain/lambdas/search-index-consumer/package.json @@ -10,6 +10,7 @@ "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", + "@tech-matters/s3-client": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", "@tech-matters/types": "^1.0.0" From d5ec170e72e75fdbdaae05dce68e84e5e80efc79 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 7 May 2024 18:36:39 -0300 Subject: [PATCH 08/55] chore: fix TS complaints --- .../convertToIndexDocument.ts | 30 +++++++++---------- hrm-domain/packages/hrm-types/Case.ts | 17 ++++++++--- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 42d9cdbca..5cc1673f1 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -107,17 +107,17 @@ const convertToContactDocument = ({ id, compundId, createdAt, - updatedAt, - createdBy, - updatedBy, + updatedAt: updatedAt ?? '', + createdBy: createdBy ?? '', + updatedBy: updatedBy ?? '', finalized: Boolean(finalizedAt), - helpline, - channel, - number, - timeOfContact, + helpline: helpline ?? '', + channel: channel ?? '', + number: number ?? '', + timeOfContact: timeOfContact ?? '', transcript, - twilioWorkerId, - content: typeof rawJson === 'object' ? JSON.stringify(rawJson) : rawJson, + twilioWorkerId: twilioWorkerId ?? '', + content: JSON.stringify(rawJson) ?? '', join_field: { name: 'contact', ...(caseId && { parent: `case_${caseId}` }) }, high_boost_global: '', // highBoostGlobal.join(' '), low_boost_global: '', // lowBoostGlobal.join(' '), @@ -151,9 +151,9 @@ const convertToCaseDocument = ({ sectionsArray.map(section => ({ accountSid: accountSid as string, createdAt: section.createdAt, - updatedAt: section.updatedAt, createdBy: section.createdBy, - updatedBy: section.updatedBy, + updatedAt: section.updatedAt ?? '', + updatedBy: section.updatedBy ?? '', sectionId: section.sectionId, sectionType, content: @@ -174,11 +174,11 @@ const convertToCaseDocument = ({ updatedBy, helpline, twilioWorkerId, - previousStatus, status, - statusUpdatedAt, - statusUpdatedBy, - content: typeof info === 'object' ? JSON.stringify(info) : info, + previousStatus: previousStatus ?? '', + statusUpdatedAt: statusUpdatedAt ?? '', + statusUpdatedBy: statusUpdatedBy ?? '', + content: JSON.stringify(info) ?? '', sections: mappedSections, join_field: { name: 'case' }, high_boost_global: '', // highBoostGlobal.join(' '), diff --git a/hrm-domain/packages/hrm-types/Case.ts b/hrm-domain/packages/hrm-types/Case.ts index 7d07b5524..03b8ed5c2 100644 --- a/hrm-domain/packages/hrm-types/Case.ts +++ b/hrm-domain/packages/hrm-types/Case.ts @@ -62,12 +62,18 @@ const getSectionSpecificDataFromNotesOrReferrals = ( }; export const WELL_KNOWN_CASE_SECTION_NAMES = { - households: { getSectionSpecificData: s => s.household, sectionTypeName: 'household' }, + households: { + getSectionSpecificData: (s: any) => s.household, + sectionTypeName: 'household', + }, perpetrators: { - getSectionSpecificData: s => s.perpetrator, + getSectionSpecificData: (s: any) => s.perpetrator, sectionTypeName: 'perpetrator', }, - incidents: { getSectionSpecificData: s => s.incident, sectionTypeName: 'incident' }, + incidents: { + getSectionSpecificData: (s: any) => s.incident, + sectionTypeName: 'incident', + }, counsellorNotes: { getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, sectionTypeName: 'note', @@ -76,7 +82,10 @@ export const WELL_KNOWN_CASE_SECTION_NAMES = { getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, sectionTypeName: 'referral', }, - documents: { getSectionSpecificData: s => s.document, sectionTypeName: 'document' }, + documents: { + getSectionSpecificData: (s: any) => s.document, + sectionTypeName: 'document', + }, } as const; type PrecalculatedPermissions = Record<'userOwnsContact', boolean>; From 43456e22a0b1eef661316079a6d8299a132b58fc Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 8 May 2024 17:42:25 -0300 Subject: [PATCH 09/55] fix: add emtpy parent if missing in contact document --- .../lambdas/search-index-consumer/index.ts | 14 ++++++++++++-- .../hrm-search-config/convertToIndexDocument.ts | 17 +++++++++++++++-- .../elasticsearch-client/src/indexDocument.ts | 3 +++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index c635ce720..9ca426264 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -31,6 +31,7 @@ import { newErr, newOkFromData, } from '@tech-matters/types'; +import { getContactParentId } from '@tech-matters/hrm-search-config/convertToIndexDocument'; export type MessagesByAccountSid = Record< AccountSID, @@ -40,6 +41,7 @@ type PayloadWithMeta = { payload: IndexPayload; documentId: string; messageId: string; + routing?: string; }; type PayloadsByIndex = { [indexType: string]: PayloadWithMeta[]; @@ -103,7 +105,14 @@ const messagesToPayloadsByIndex = ( ...accum, [HRM_CASES_CONTACTS_INDEX_TYPE]: [ ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), - { ...currM, payload: { ...message, transcript: '' } }, + { + ...currM, + payload: { ...message, transcript: '' }, + routing: getContactParentId( + HRM_CASES_CONTACTS_INDEX_TYPE, + message.contact.caseId, + ), + }, ], }; } @@ -131,12 +140,13 @@ const indexDocumentsByIndex = ); const indexed = await Promise.all( - payloads.map(({ documentId, messageId, payload }) => + payloads.map(({ documentId, messageId, payload, routing }) => client .indexDocument({ id: documentId, document: payload, autocreate: true, + routing, }) .then(result => ({ accountSid, diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 5cc1673f1..9b3486c45 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -25,10 +25,11 @@ import { assertExhaustive, AccountSID } from '@tech-matters/types'; import type { CaseService, Contact } from '@tech-matters/hrm-types'; -import type { +import { ContactDocument, CaseDocument, CasesContactsDocument, + HRM_CASES_CONTACTS_INDEX_TYPE, } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; @@ -55,6 +56,15 @@ const getContactDocumentId = ({ contact, type }: IndexContactMessage) => const getCaseDocumentId = ({ case: caseObj, type }: IndexCaseMessage) => `${type}_${caseObj.id}`; +export const getContactParentId = ( + indexType: typeof HRM_CASES_CONTACTS_INDEX_TYPE, + parentId?: string | number, +) => { + if (indexType === HRM_CASES_CONTACTS_INDEX_TYPE) { + return parentId ? `case_${parentId}` : ''; + } +}; + export const getDocumentId = (m: IndexMessage) => { const { type } = m; switch (type) { @@ -118,7 +128,10 @@ const convertToContactDocument = ({ transcript, twilioWorkerId: twilioWorkerId ?? '', content: JSON.stringify(rawJson) ?? '', - join_field: { name: 'contact', ...(caseId && { parent: `case_${caseId}` }) }, + join_field: { + name: 'contact', + parent: getContactParentId(HRM_CASES_CONTACTS_INDEX_TYPE, caseId), + }, high_boost_global: '', // highBoostGlobal.join(' '), low_boost_global: '', // lowBoostGlobal.join(' '), }; diff --git a/packages/elasticsearch-client/src/indexDocument.ts b/packages/elasticsearch-client/src/indexDocument.ts index a1811688a..8b098f070 100644 --- a/packages/elasticsearch-client/src/indexDocument.ts +++ b/packages/elasticsearch-client/src/indexDocument.ts @@ -21,6 +21,7 @@ export type IndexDocumentExtraParams = { id: string; document: T; autocreate?: boolean; + routing?: string; }; export type IndexDocumentParams = PassThroughConfig & IndexDocumentExtraParams; @@ -33,6 +34,7 @@ export const indexDocument = async ({ index, indexConfig, autocreate = false, + routing = undefined, }: IndexDocumentParams): Promise => { if (autocreate) { // const exists = await client.indices.exists({ index }); @@ -46,6 +48,7 @@ export const indexDocument = async ({ index, id, document: convertedDocument, + ...(routing && { routing }), }); }; From 83c177748addc0d5b1a9ea4fb09edad326ee29ca Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 8 May 2024 18:03:21 -0300 Subject: [PATCH 10/55] fix: wrong import --- hrm-domain/lambdas/search-index-consumer/index.ts | 2 +- hrm-domain/packages/hrm-search-config/index.ts | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 9ca426264..ec93a2c06 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -23,6 +23,7 @@ import { hrmIndexConfiguration, IndexPayload, getDocumentId, + getContactParentId, } from '@tech-matters/hrm-search-config'; import { AccountSID, @@ -31,7 +32,6 @@ import { newErr, newOkFromData, } from '@tech-matters/types'; -import { getContactParentId } from '@tech-matters/hrm-search-config/convertToIndexDocument'; export type MessagesByAccountSid = Record< AccountSID, diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index b8a829737..41eb6c363 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -25,7 +25,12 @@ export { HRM_CASES_CONTACTS_INDEX_TYPE, CasesContactsDocument, } from './hrmIndexDocumentMappings'; -export { IndexMessage, IndexPayload, getDocumentId } from './convertToIndexDocument'; +export { + IndexMessage, + IndexPayload, + getDocumentId, + getContactParentId, +} from './convertToIndexDocument'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { From 941756786c99523434165d32d650e7069a722dc2 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 13:47:22 -0300 Subject: [PATCH 11/55] chore: added logic to update documents via elasticsearch client --- packages/elasticsearch-client/src/client.ts | 13 +++ .../elasticsearch-client/src/config/index.ts | 4 +- .../src/config/indexConfiguration.ts | 14 ++- .../elasticsearch-client/src/indexDocument.ts | 5 +- .../src/updateDocument.ts | 88 +++++++++++++++++++ 5 files changed, 116 insertions(+), 8 deletions(-) create mode 100644 packages/elasticsearch-client/src/updateDocument.ts diff --git a/packages/elasticsearch-client/src/client.ts b/packages/elasticsearch-client/src/client.ts index 91963f7fd..0286d781d 100644 --- a/packages/elasticsearch-client/src/client.ts +++ b/packages/elasticsearch-client/src/client.ts @@ -23,6 +23,13 @@ import { IndexDocumentExtraParams, IndexDocumentResponse, } from './indexDocument'; +import { + updateDocument, + UpdateDocumentExtraParams, + UpdateDocumentResponse, + updateScript, + UpdateScriptExtraParams, +} from './updateDocument'; import getAccountSid from './getAccountSid'; import { search, SearchExtraParams } from './search'; import { suggest, SuggestExtraParams } from './suggest'; @@ -92,6 +99,8 @@ const getEsConfig = async ({ */ export type IndexClient = { indexDocument: (args: IndexDocumentExtraParams) => Promise; + updateDocument: (args: UpdateDocumentExtraParams) => Promise; + updateScript: (args: UpdateScriptExtraParams) => Promise; refreshIndex: () => Promise; executeBulk: (args: ExecuteBulkExtraParams) => Promise; createIndex: (args: CreateIndexExtraParams) => Promise; @@ -133,6 +142,10 @@ const getClientOrMock = async ({ config, index, indexType }: GetClientOrMockArgs deleteIndex: () => deleteIndex(passThroughConfig), indexDocument: (args: IndexDocumentExtraParams) => indexDocument({ ...passThroughConfig, ...args }), + updateDocument: (args: UpdateDocumentExtraParams) => + updateDocument({ ...passThroughConfig, ...args }), + updateScript: (args: UpdateScriptExtraParams) => + updateScript({ ...passThroughConfig, ...args }), executeBulk: (args: ExecuteBulkExtraParams) => executeBulk({ ...passThroughConfig, ...args }), }; diff --git a/packages/elasticsearch-client/src/config/index.ts b/packages/elasticsearch-client/src/config/index.ts index ba0319010..421aa4f10 100644 --- a/packages/elasticsearch-client/src/config/index.ts +++ b/packages/elasticsearch-client/src/config/index.ts @@ -17,6 +17,6 @@ export * from './indexConfiguration'; export * from './searchConfiguration'; export type CreateIndexConvertedDocument = { - high_boost_global: string; - low_boost_global: string; + high_boost_global?: string; + low_boost_global?: string; } & TDoc; diff --git a/packages/elasticsearch-client/src/config/indexConfiguration.ts b/packages/elasticsearch-client/src/config/indexConfiguration.ts index f6a5bfb6f..b52a3fae6 100644 --- a/packages/elasticsearch-client/src/config/indexConfiguration.ts +++ b/packages/elasticsearch-client/src/config/indexConfiguration.ts @@ -14,10 +14,20 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesCreateRequest, Script } from '@elastic/elasticsearch/lib/api/types'; import { CreateIndexConvertedDocument } from './index'; export type IndexConfiguration = { getCreateIndexParams: (indexName: string) => IndicesCreateRequest; - convertToIndexDocument: (sourceEntity: T) => CreateIndexConvertedDocument; + convertToIndexDocument: ( + sourceEntity: T, + indexName: string, + ) => CreateIndexConvertedDocument; + convertToScriptUpdate?: ( + sourceEntity: T, + indexName: string, + ) => { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; + }; }; diff --git a/packages/elasticsearch-client/src/indexDocument.ts b/packages/elasticsearch-client/src/indexDocument.ts index 8b098f070..12c7b5e54 100644 --- a/packages/elasticsearch-client/src/indexDocument.ts +++ b/packages/elasticsearch-client/src/indexDocument.ts @@ -21,7 +21,6 @@ export type IndexDocumentExtraParams = { id: string; document: T; autocreate?: boolean; - routing?: string; }; export type IndexDocumentParams = PassThroughConfig & IndexDocumentExtraParams; @@ -34,7 +33,6 @@ export const indexDocument = async ({ index, indexConfig, autocreate = false, - routing = undefined, }: IndexDocumentParams): Promise => { if (autocreate) { // const exists = await client.indices.exists({ index }); @@ -42,13 +40,12 @@ export const indexDocument = async ({ await createIndex({ client, index, indexConfig }); } - const convertedDocument = indexConfig.convertToIndexDocument(document); + const convertedDocument = indexConfig.convertToIndexDocument(document, index); return client.index({ index, id, document: convertedDocument, - ...(routing && { routing }), }); }; diff --git a/packages/elasticsearch-client/src/updateDocument.ts b/packages/elasticsearch-client/src/updateDocument.ts new file mode 100644 index 000000000..7dba5b051 --- /dev/null +++ b/packages/elasticsearch-client/src/updateDocument.ts @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ +import { UpdateResponse } from '@elastic/elasticsearch/lib/api/types'; +import { PassThroughConfig } from './client'; +import createIndex from './createIndex'; + +type UpdateParams = { id: string; document: T; autocreate?: boolean }; + +export type UpdateDocumentExtraParams = UpdateParams & { + docAsUpsert?: boolean; +}; + +export type UpdateDocumentParams = PassThroughConfig & UpdateDocumentExtraParams; +export type UpdateDocumentResponse = UpdateResponse; + +export const updateDocument = async ({ + client, + document, + id, + index, + indexConfig, + docAsUpsert = false, + autocreate = false, +}: UpdateDocumentParams): Promise => { + if (docAsUpsert && autocreate) { + // const exists = await client.indices.exists({ index }); + // NOTE: above check is already performed in createIndex + await createIndex({ client, index, indexConfig }); + } + + const documentUpdate = indexConfig.convertToIndexDocument(document, index); + + return client.update({ + index, + id, + doc: documentUpdate, + doc_as_upsert: docAsUpsert, + }); +}; + +export type UpdateScriptExtraParams = UpdateParams & { scriptedUpsert?: boolean }; +export type UpdateScriptParams = PassThroughConfig & UpdateScriptExtraParams; + +export const updateScript = async ({ + client, + document, + id, + index, + indexConfig, + scriptedUpsert = false, + autocreate = false, +}: UpdateScriptParams): Promise => { + if (!indexConfig.convertToScriptUpdate) { + throw new Error(`updateScript error: convertToScriptDocument not provided`); + } + + if (scriptedUpsert && autocreate) { + // const exists = await client.indices.exists({ index }); + // NOTE: above check is already performed in createIndex + await createIndex({ client, index, indexConfig }); + } + + const { documentUpdate, scriptUpdate } = indexConfig.convertToScriptUpdate( + document, + index, + ); + + return client.update({ + index, + id, + script: scriptUpdate, + upsert: documentUpdate, + scripted_upsert: scriptedUpsert, + }); +}; From 2bc2d8e3e110348f2eac683754878fbb9da735a1 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 14:25:31 -0300 Subject: [PATCH 12/55] chore: update hrm-search-config package to generate case update scripts based off contacts --- .../convertToIndexDocument.ts | 234 +++++++++++------- .../hrm-search-config/getCreateIndexParams.ts | 37 ++- .../hrmIndexDocumentMappings/index.ts | 64 ++--- .../{mappingCasesContacts.ts => mappings.ts} | 25 +- .../packages/hrm-search-config/index.ts | 18 +- 5 files changed, 224 insertions(+), 154 deletions(-) rename hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/{mappingCasesContacts.ts => mappings.ts} (85%) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 9b3486c45..d9bcb6b4f 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,26 +23,32 @@ * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html */ -import { assertExhaustive, AccountSID } from '@tech-matters/types'; +import type { Script } from '@elastic/elasticsearch/lib/api/types'; import type { CaseService, Contact } from '@tech-matters/hrm-types'; +import { assertExhaustive, AccountSID } from '@tech-matters/types'; import { ContactDocument, CaseDocument, - CasesContactsDocument, - HRM_CASES_CONTACTS_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, + HRM_CASES_INDEX_TYPE, } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; +type IndexOperation = 'index' | 'remove'; + type IndexContactMessage = { type: 'contact'; - contact: Contact; + operation: IndexOperation; + contact: Pick & Partial; }; type IndexCaseMessage = { type: 'case'; - case: Omit & { - sections: NonNullable; - }; + operation: IndexOperation; + case: Pick & + Partial> & { + sections: NonNullable; + }; }; export type IndexMessage = { accountSid: AccountSID } & ( @@ -50,36 +56,6 @@ export type IndexMessage = { accountSid: AccountSID } & ( | IndexCaseMessage ); -const getContactDocumentId = ({ contact, type }: IndexContactMessage) => - `${type}_${contact.id}`; - -const getCaseDocumentId = ({ case: caseObj, type }: IndexCaseMessage) => - `${type}_${caseObj.id}`; - -export const getContactParentId = ( - indexType: typeof HRM_CASES_CONTACTS_INDEX_TYPE, - parentId?: string | number, -) => { - if (indexType === HRM_CASES_CONTACTS_INDEX_TYPE) { - return parentId ? `case_${parentId}` : ''; - } -}; - -export const getDocumentId = (m: IndexMessage) => { - const { type } = m; - switch (type) { - case 'contact': { - return getContactDocumentId(m); - } - case 'case': { - return getCaseDocumentId(m); - } - default: { - return assertExhaustive(type); - } - } -}; - type IndexPayloadContact = IndexContactMessage & { transcript: NonNullable; }; @@ -88,8 +64,16 @@ type IndexPayloadCase = IndexCaseMessage; export type IndexPayload = IndexPayloadContact | IndexPayloadCase; -const convertToContactDocument = ({ - type, +const filterEmpty = (doc: T): T => + Object.entries(doc).reduce((accum, [key, value]) => { + if (value) { + return { ...accum, [key]: value }; + } + + return accum; + }, {} as T); + +const convertContactToContactDocument = ({ contact, transcript, }: IndexPayloadContact): CreateIndexConvertedDocument => { @@ -102,43 +86,36 @@ const convertToContactDocument = ({ updatedBy, finalizedAt, helpline, - caseId, number, channel, timeOfContact, twilioWorkerId, rawJson, } = contact; - const compundId = getContactDocumentId({ type, contact }); - return { - type, + const contactDocument: ContactDocument = { accountSid, id, - compundId, createdAt, - updatedAt: updatedAt ?? '', - createdBy: createdBy ?? '', - updatedBy: updatedBy ?? '', + updatedAt: updatedAt, + createdBy: createdBy, + updatedBy: updatedBy, finalized: Boolean(finalizedAt), - helpline: helpline ?? '', - channel: channel ?? '', - number: number ?? '', - timeOfContact: timeOfContact ?? '', + helpline: helpline, + channel: channel, + number: number, + timeOfContact: timeOfContact, transcript, - twilioWorkerId: twilioWorkerId ?? '', - content: JSON.stringify(rawJson) ?? '', - join_field: { - name: 'contact', - parent: getContactParentId(HRM_CASES_CONTACTS_INDEX_TYPE, caseId), - }, - high_boost_global: '', // highBoostGlobal.join(' '), - low_boost_global: '', // lowBoostGlobal.join(' '), + twilioWorkerId: twilioWorkerId, + content: JSON.stringify(rawJson), + // high_boost_global: '', // highBoostGlobal.join(' '), + // low_boost_global: '', // lowBoostGlobal.join(' '), }; + + return filterEmpty(contactDocument); }; -const convertToCaseDocument = ({ - type, +const convertCaseToCaseDocument = ({ case: caseObj, }: IndexPayloadCase): CreateIndexConvertedDocument => { const { @@ -157,16 +134,14 @@ const convertToCaseDocument = ({ sections, info, } = caseObj; - const compundId = getCaseDocumentId({ type, case: caseObj }); - const mappedSections: CaseDocument['sections'] = Object.entries(sections).flatMap( ([sectionType, sectionsArray]) => sectionsArray.map(section => ({ accountSid: accountSid as string, createdAt: section.createdAt, createdBy: section.createdBy, - updatedAt: section.updatedAt ?? '', - updatedBy: section.updatedBy ?? '', + updatedAt: section.updatedAt, + updatedBy: section.updatedBy, sectionId: section.sectionId, sectionType, content: @@ -176,11 +151,9 @@ const convertToCaseDocument = ({ })), ); - return { - type, + const caseDocument: CaseDocument = { accountSid, id, - compundId, createdAt, updatedAt, createdBy, @@ -188,30 +161,125 @@ const convertToCaseDocument = ({ helpline, twilioWorkerId, status, - previousStatus: previousStatus ?? '', - statusUpdatedAt: statusUpdatedAt ?? '', - statusUpdatedBy: statusUpdatedBy ?? '', - content: JSON.stringify(info) ?? '', + previousStatus: previousStatus, + statusUpdatedAt: statusUpdatedAt, + statusUpdatedBy: statusUpdatedBy, + content: JSON.stringify(info), sections: mappedSections, - join_field: { name: 'case' }, - high_boost_global: '', // highBoostGlobal.join(' '), - low_boost_global: '', // lowBoostGlobal.join(' '), + contacts: null, + // high_boost_global: '', // highBoostGlobal.join(' '), + // low_boost_global: '', // lowBoostGlobal.join(' '), }; + + return filterEmpty(caseDocument); +}; + +const convertToContactIndexDocument = (payload: IndexPayload) => { + if (payload.type === 'contact') { + return convertContactToContactDocument(payload); + } + + throw new Error( + `convertToContactIndexDocument not implemented for type ${payload.type} and operation ${payload.operation}`, + ); +}; + +const convertToCaseIndexDocument = (payload: IndexPayload) => { + if (payload.type === 'case') { + return convertCaseToCaseDocument(payload); + } + + throw new Error( + `convertToCaseIndexDocument not implemented for type ${payload.type} and operation ${payload.operation}`, + ); }; export const convertToIndexDocument = ( payload: IndexPayload, -): CreateIndexConvertedDocument => { - const { type } = payload; - switch (type) { - case 'contact': { - return convertToContactDocument(payload); + indexName: string, +): CreateIndexConvertedDocument => { + if (indexName.endsWith(HRM_CONTACTS_INDEX_TYPE)) { + return convertToContactIndexDocument(payload); + } + + if (indexName.endsWith(HRM_CASES_INDEX_TYPE)) { + return convertToCaseIndexDocument(payload); + } + + throw new Error(`convertToIndexDocument not implemented for index ${indexName}`); +}; + +const convertContactToCaseScriptUpdate = ( + payload: IndexPayloadContact, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + const { operation } = payload; + const { accountSid, caseId } = payload.contact; + + switch (operation) { + case 'index': { + const contactDocument = convertContactToContactDocument(payload); + + const documentUpdate: CreateIndexConvertedDocument = { + id: parseInt(caseId, 10), + accountSid, + contacts: [contactDocument], + }; + + const scriptUpdate: Script = { + source: + 'def replaceContact(Map newContact, List contacts) { contacts.removeIf(contact -> contact.id == newContact.id); contacts.add(newContact); } replaceContact(params.newContact, ctx._source.contacts);', + params: { + newContact: contactDocument, + }, + }; + + return { documentUpdate, scriptUpdate }; } - case 'case': { - return convertToCaseDocument(payload); + case 'remove': { + const scriptUpdate: Script = { + source: + 'def removeContact(int contactId, List contacts) { contacts.removeIf(contact -> contact.id == contactId); } removeContact(params.contactId, ctx._source.contacts);', + params: { + contactId: payload.contact.id, + }, + }; + + return { documentUpdate: undefined, scriptUpdate }; } default: { - return assertExhaustive(type); + return assertExhaustive(operation); } } }; + +const convertToCaseScriptUpdate = ( + payload: IndexPayload, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + if (payload.type === 'contact') { + return convertContactToCaseScriptUpdate(payload); + } + + throw new Error( + `convertToCaseScriptDocument not implemented for type ${payload.type} and operation ${payload.operation}`, + ); +}; + +export const convertToScriptUpdate = ( + payload: IndexPayload, + indexName: string, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + if (indexName.endsWith(HRM_CASES_INDEX_TYPE)) { + return convertToCaseScriptUpdate(payload); + } + + throw new Error(`convertToScriptDocument not implemented for index ${indexName}`); +}; diff --git a/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts b/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts index cd20c81d8..5850c6cd1 100644 --- a/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts +++ b/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts @@ -26,11 +26,13 @@ import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types'; import { - mappingCasesContacts, - HRM_CASES_CONTACTS_INDEX_TYPE, + HRM_CASES_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, + caseMapping, + contactMapping, } from './hrmIndexDocumentMappings'; -const getCreateHrmCasesContactsIndexParams = (index: string): IndicesCreateRequest => { +const getCreateHrmContactsIndexParams = (index: string): IndicesCreateRequest => { return { index, // settings: { @@ -43,7 +45,26 @@ const getCreateHrmCasesContactsIndexParams = (index: string): IndicesCreateReque low_boost_global: { type: 'text', }, - ...mappingCasesContacts, + ...contactMapping, + }, + }, + }; +}; + +const getCreateHrmCaseIndexParams = (index: string): IndicesCreateRequest => { + return { + index, + // settings: { + // }, + mappings: { + properties: { + high_boost_global: { + type: 'text', + }, + low_boost_global: { + type: 'text', + }, + ...caseMapping, }, }, }; @@ -54,8 +75,12 @@ const getCreateHrmCasesContactsIndexParams = (index: string): IndicesCreateReque * @param index */ export const getCreateIndexParams = (index: string): IndicesCreateRequest => { - if (index.endsWith(HRM_CASES_CONTACTS_INDEX_TYPE)) { - return getCreateHrmCasesContactsIndexParams(index); + if (index.endsWith(HRM_CONTACTS_INDEX_TYPE)) { + return getCreateHrmContactsIndexParams(index); + } + + if (index.endsWith(HRM_CASES_INDEX_TYPE)) { + return getCreateHrmCaseIndexParams(index); } throw new Error(`getCreateIndexParams not implemented for index ${index}`); diff --git a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts index 7f4d638a2..52962c84b 100644 --- a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts +++ b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts @@ -15,46 +15,36 @@ */ import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; -import { caseMapping, contactMapping } from './mappingCasesContacts'; +import { caseMapping, contactMapping } from './mappings'; -export { mapping as mappingCasesContacts } from './mappingCasesContacts'; +export { caseMapping, contactMapping } from './mappings'; -type MappingToDocument>> = { - [k in keyof T]: k extends string - ? T[k]['type'] extends 'keyword' - ? string - : T[k]['type'] extends 'text' - ? string - : T[k]['type'] extends 'integer' - ? number - : T[k]['type'] extends 'boolean' - ? boolean - : T[k]['type'] extends 'date' - ? string - : T[k]['type'] extends 'join' - ? { name: string; parent?: string } - : T[k]['type'] extends 'nested' - ? T[k] extends { - properties: Record; - } - ? MappingToDocument[] - : never - : never // forbid non-used types to force proper implementation - : never; -}; - -export type ContactDocument = MappingToDocument & - NonNullable<{ - type: 'contact'; - join_field: NonNullable<{ name: 'contact'; parent?: string }>; +export type MappingToDocument>> = + Partial<{ + [k in keyof T]: k extends string + ? T[k]['type'] extends 'keyword' + ? string | null + : T[k]['type'] extends 'text' + ? string | null + : T[k]['type'] extends 'integer' + ? number | null + : T[k]['type'] extends 'boolean' + ? boolean | null + : T[k]['type'] extends 'date' + ? string | null + : T[k]['type'] extends 'nested' + ? T[k] extends { + properties: Record; + } + ? MappingToDocument[] + : never + : never // forbid non-used types to force proper implementation + : never; }>; -export type CaseDocument = MappingToDocument & - NonNullable<{ - type: 'case'; - join_field: NonNullable<{ name: 'case' }>; - }>; +export type ContactDocument = MappingToDocument; -export type CasesContactsDocument = ContactDocument | CaseDocument; +export type CaseDocument = MappingToDocument; -export const HRM_CASES_CONTACTS_INDEX_TYPE = 'hrm-cases-contacts'; +export const HRM_CONTACTS_INDEX_TYPE = 'hrm-contacts'; +export const HRM_CASES_INDEX_TYPE = 'hrm-cases'; diff --git a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappingCasesContacts.ts b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappings.ts similarity index 85% rename from hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappingCasesContacts.ts rename to hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappings.ts index ef810fde3..fd4dd1006 100644 --- a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappingCasesContacts.ts +++ b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappings.ts @@ -14,8 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; - // Properties present in root and nested documents const commonProperties = { accountSid: { @@ -40,15 +38,9 @@ const commonProperties = { // Properties shared by both types of documents, cases and contacts const rootProperties = { - type: { - type: 'keyword', - }, id: { type: 'integer', }, - compundId: { - type: 'keyword', - }, twilioWorkerId: { type: 'keyword', }, @@ -56,12 +48,6 @@ const rootProperties = { type: 'keyword', }, ...commonProperties, - join_field: { - type: 'join', - relations: { - case: 'contact', - }, - }, } as const; // Properties specific to contacts @@ -111,9 +97,10 @@ export const caseMapping = { ...commonProperties, }, }, -} as const; - -export const mapping: Record = { - ...contactMapping, - ...caseMapping, + contacts: { + type: 'nested', + properties: { + ...contactMapping, + }, + }, } as const; diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index 41eb6c363..c4c76df56 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -14,7 +14,11 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { IndexPayload, convertToIndexDocument } from './convertToIndexDocument'; +import { + IndexPayload, + convertToIndexDocument, + convertToScriptUpdate, +} from './convertToIndexDocument'; import { getCreateIndexParams } from './getCreateIndexParams'; import type { IndexConfiguration, @@ -22,15 +26,10 @@ import type { } from '@tech-matters/elasticsearch-client'; export { - HRM_CASES_CONTACTS_INDEX_TYPE, - CasesContactsDocument, + HRM_CASES_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, } from './hrmIndexDocumentMappings'; -export { - IndexMessage, - IndexPayload, - getDocumentId, - getContactParentId, -} from './convertToIndexDocument'; +export { IndexMessage, IndexPayload } from './convertToIndexDocument'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { @@ -47,5 +46,6 @@ export const hrmSearchConfiguration: SearchConfiguration = { export const hrmIndexConfiguration: IndexConfiguration = { convertToIndexDocument, + convertToScriptUpdate, getCreateIndexParams, }; From 340bdd24715825cf9e50fee1af73c8bf746ef655 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 14:26:02 -0300 Subject: [PATCH 13/55] chore: search-index-consumer rework --- .../lambdas/search-index-consumer/index.ts | 223 ++++++++++-------- 1 file changed, 128 insertions(+), 95 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index ec93a2c06..d6b405a5a 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -14,16 +14,15 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ import type { SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda'; -import { getClient } from '@tech-matters/elasticsearch-client'; +import { getClient, IndexClient } from '@tech-matters/elasticsearch-client'; // import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; import { HrmIndexProcessorError } from '@tech-matters/job-errors'; import { - IndexMessage, - HRM_CASES_CONTACTS_INDEX_TYPE, + HRM_CASES_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, hrmIndexConfiguration, + IndexMessage, IndexPayload, - getDocumentId, - getContactParentId, } from '@tech-matters/hrm-search-config'; import { AccountSID, @@ -33,66 +32,38 @@ import { newOkFromData, } from '@tech-matters/types'; -export type MessagesByAccountSid = Record< - AccountSID, - { message: IndexMessage; documentId: string; messageId: string }[] ->; +type MessageWithMeta = { message: IndexMessage; messageId: string }; +type MessagesByAccountSid = Record; type PayloadWithMeta = { + documentId: number; payload: IndexPayload; - documentId: string; messageId: string; - routing?: string; + indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript'; }; type PayloadsByIndex = { [indexType: string]: PayloadWithMeta[]; }; -export type PayloadsByAccountSid = Record; +type PayloadsByAccountSid = Record; -type MessagesByDocumentId = { - [documentId: string]: { - documentId: string; - messageId: string; - message: IndexMessage; - }; -}; - -const reduceByDocumentId = ( - accum: MessagesByDocumentId, +const groupMessagesByAccountSid = ( + accum: MessagesByAccountSid, curr: SQSRecord, -): MessagesByDocumentId => { +): MessagesByAccountSid => { const { messageId, body } = curr; const message = JSON.parse(body) as IndexMessage; - const documentId = getDocumentId(message); - - return { ...accum, [documentId]: { documentId, messageId, message } }; -}; - -const groupMessagesByAccountSid = ( - accum: MessagesByAccountSid, - curr: { - documentId: string; - messageId: string; - message: IndexMessage; - }, -): MessagesByAccountSid => { - const { message } = curr; const { accountSid } = message; if (!accum[accountSid]) { - return { ...accum, [accountSid]: [curr] }; + return { ...accum, [accountSid]: [{ messageId, message }] }; } - return { ...accum, [accountSid]: [...accum[accountSid], curr] }; + return { ...accum, [accountSid]: [...accum[accountSid], { messageId, message }] }; }; const messagesToPayloadsByIndex = ( accum: PayloadsByIndex, - currM: { - documentId: string; - messageId: string; - message: IndexMessage; - }, + currM: MessageWithMeta, ): PayloadsByIndex => { const { message } = currM; @@ -103,25 +74,42 @@ const messagesToPayloadsByIndex = ( // TODO: Pull the transcripts from S3 (if any) return { ...accum, - [HRM_CASES_CONTACTS_INDEX_TYPE]: [ - ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), + // add an upsert job to HRM_CONTACTS_INDEX_TYPE index + [HRM_CONTACTS_INDEX_TYPE]: [ + ...(accum[HRM_CONTACTS_INDEX_TYPE] ?? []), { ...currM, + documentId: message.contact.id, payload: { ...message, transcript: '' }, - routing: getContactParentId( - HRM_CASES_CONTACTS_INDEX_TYPE, - message.contact.caseId, - ), + indexHandler: 'updateDocument', }, ], + // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: message.contact.caseId + ? [ + ...(accum[HRM_CASES_INDEX_TYPE] ?? []), + { + ...currM, + documentId: parseInt(message.contact.caseId, 10), + payload: { ...message, transcript: '' }, + indexHandler: 'updateScript', + }, + ] + : accum[HRM_CASES_INDEX_TYPE], }; } case 'case': { return { ...accum, - [HRM_CASES_CONTACTS_INDEX_TYPE]: [ - ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), - { ...currM, payload: { ...message } }, + // add an upsert job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: [ + ...(accum[HRM_CASES_INDEX_TYPE] ?? []), + { + ...currM, + documentId: message.case.id, + payload: { ...message }, + indexHandler: 'updateDocument', + }, ], }; } @@ -131,50 +119,97 @@ const messagesToPayloadsByIndex = ( } }; -const indexDocumentsByIndex = - (accountSid: string) => - async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { - // get the client for the accountSid-indexType pair - const client = (await getClient({ accountSid, indexType })).indexClient( - hrmIndexConfiguration, - ); +const handleIndexPayload = + ({ + accountSid, + client, + indexType, + }: { + accountSid: string; + client: IndexClient; + indexType: string; + }) => + async ({ documentId, indexHandler, messageId, payload }: PayloadWithMeta) => { + try { + switch (indexHandler) { + case 'indexDocument': { + const result = await client.indexDocument({ + id: documentId.toString(), + document: payload, + autocreate: true, + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + case 'updateDocument': { + const result = await client.updateDocument({ + id: documentId.toString(), + document: payload, + autocreate: true, + docAsUpsert: true, + }); - const indexed = await Promise.all( - payloads.map(({ documentId, messageId, payload, routing }) => - client - .indexDocument({ - id: documentId, + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + case 'updateScript': { + const result = await client.updateScript({ document: payload, + id: documentId.toString(), autocreate: true, - routing, - }) - .then(result => ({ + scriptedUpsert: true, + }); + + return { accountSid, indexType, - documentId, messageId, result: newOkFromData(result), - })) - .catch(err => { - console.error( - new HrmIndexProcessorError('Failed to process search index request'), - err, - ); - - return { - accountSid, - indexType, - documentId, - messageId, - result: newErr({ - error: 'ErrorFailedToInex', - message: err instanceof Error ? err.message : String(err), - }), - }; - }), - ), + }; + } + default: { + return assertExhaustive(indexHandler); + } + } + } catch (err) { + console.error( + new HrmIndexProcessorError('handleIndexPayload: Failed to process index request'), + err, + ); + + return { + accountSid, + indexType, + messageId, + result: newErr({ + error: 'ErrorFailedToInex', + message: err instanceof Error ? err.message : String(err), + }), + }; + } + }; + +const indexDocumentsByIndex = + (accountSid: string) => + async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { + // get the client for the accountSid-indexType pair + const client = (await getClient({ accountSid, indexType })).indexClient( + hrmIndexConfiguration, ); + const mapper = handleIndexPayload({ client, accountSid, indexType }); + + const indexed = await Promise.all(payloads.map(mapper)); + return indexed; }; @@ -193,13 +228,11 @@ export const handler = async (event: SQSEvent): Promise => { console.debug('Received event:', JSON.stringify(event, null, 2)); try { - // link each composite "documentId" to it's corresponding "messageId" - const documentIdToMessage = event.Records.reduce(reduceByDocumentId, {}); - - // group the messages by accountSid - const messagesByAccoundSid = Object.values( - documentIdToMessage, - ).reduce(groupMessagesByAccountSid, {}); + // group the messages by accountSid while adding message meta + const messagesByAccoundSid = event.Records.reduce( + groupMessagesByAccountSid, + {}, + ); // generate corresponding IndexPayload for each IndexMessage and group them by target indexType const documentsByAccountSid: PayloadsByAccountSid = Object.fromEntries( From 7cf998ce812a4ca6a5244e9d8d0a1898538d4f6a Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 14:33:28 -0300 Subject: [PATCH 14/55] fix: small TS errors --- .../packages/hrm-search-config/convertToIndexDocument.ts | 4 ++-- .../hrm-search-config/hrmIndexDocumentMappings/index.ts | 2 +- packages/elasticsearch-client/src/executeBulk.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index d9bcb6b4f..2226cf2e2 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -223,7 +223,7 @@ const convertContactToCaseScriptUpdate = ( const contactDocument = convertContactToContactDocument(payload); const documentUpdate: CreateIndexConvertedDocument = { - id: parseInt(caseId, 10), + id: parseInt(caseId!, 10), accountSid, contacts: [contactDocument], }; @@ -247,7 +247,7 @@ const convertContactToCaseScriptUpdate = ( }, }; - return { documentUpdate: undefined, scriptUpdate }; + return { documentUpdate: {}, scriptUpdate }; } default: { return assertExhaustive(operation); diff --git a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts index 52962c84b..87ea5e883 100644 --- a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts +++ b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts @@ -36,7 +36,7 @@ export type MappingToDocument; } - ? MappingToDocument[] + ? MappingToDocument[] | null : never : never // forbid non-used types to force proper implementation : never; diff --git a/packages/elasticsearch-client/src/executeBulk.ts b/packages/elasticsearch-client/src/executeBulk.ts index 07c701766..3e3f2b579 100644 --- a/packages/elasticsearch-client/src/executeBulk.ts +++ b/packages/elasticsearch-client/src/executeBulk.ts @@ -51,7 +51,7 @@ export const executeBulk = async ({ } else { return [ { index: { _index: index, _id: documentItem.id } }, - indexConfig.convertToIndexDocument(documentItem.document), + indexConfig.convertToIndexDocument(documentItem.document, index), ]; } }, From f5b3bef688fae6729fd976f2662200d6dd38bfc7 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 18:00:03 -0300 Subject: [PATCH 15/55] debug --- hrm-domain/lambdas/search-index-consumer/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index d6b405a5a..82c78c577 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -217,6 +217,8 @@ const indexDocumentsByAccount = async ([accountSid, payloadsByIndex]: [ string, PayloadsByIndex, ]) => { + console.log('>> indexDocumentsByAccount payloadsByIndex ', payloadsByIndex); + console.log('>> indexDocumentsByAccount: ', Object.entries(payloadsByIndex)); const resultsByIndex = await Promise.all( Object.entries(payloadsByIndex).map(indexDocumentsByIndex(accountSid)), ); From c0cabb15b28f904965e197d3368eda5fc77626e8 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 18:08:29 -0300 Subject: [PATCH 16/55] fix: contemplate case where case index payloads is undefined --- hrm-domain/lambdas/search-index-consumer/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 82c78c577..1ce9daa35 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -95,7 +95,7 @@ const messagesToPayloadsByIndex = ( indexHandler: 'updateScript', }, ] - : accum[HRM_CASES_INDEX_TYPE], + : accum[HRM_CASES_INDEX_TYPE] ?? [], }; } case 'case': { @@ -217,8 +217,6 @@ const indexDocumentsByAccount = async ([accountSid, payloadsByIndex]: [ string, PayloadsByIndex, ]) => { - console.log('>> indexDocumentsByAccount payloadsByIndex ', payloadsByIndex); - console.log('>> indexDocumentsByAccount: ', Object.entries(payloadsByIndex)); const resultsByIndex = await Promise.all( Object.entries(payloadsByIndex).map(indexDocumentsByIndex(accountSid)), ); From 8097a42de5a6192720440d0483080ff35efbecc6 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 13 May 2024 16:26:11 -0300 Subject: [PATCH 17/55] chore: fix unit tests --- .../tests/unit/convertDocumentResources.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/resources-domain/packages/resources-search-config/tests/unit/convertDocumentResources.test.ts b/resources-domain/packages/resources-search-config/tests/unit/convertDocumentResources.test.ts index ba163ee5b..81c692684 100644 --- a/resources-domain/packages/resources-search-config/tests/unit/convertDocumentResources.test.ts +++ b/resources-domain/packages/resources-search-config/tests/unit/convertDocumentResources.test.ts @@ -15,7 +15,7 @@ */ import { FlatResource } from '@tech-matters/types'; -import { resourceIndexConfiguration } from '../../index'; +import { resourceIndexConfiguration, RESOURCE_INDEX_TYPE } from '../../index'; const BASELINE_DATE = new Date('2021-01-01T00:00:00.000Z'); @@ -53,7 +53,10 @@ describe('convertIndexDocument', () => { ], }; - const document = resourceIndexConfiguration.convertToIndexDocument(resource); + const document = resourceIndexConfiguration.convertToIndexDocument( + resource, + RESOURCE_INDEX_TYPE, + ); expect(document).toEqual({ id: '1234', From a10662dbb048f82272f0af3ac3ee7ba03ee8493b Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 13 May 2024 17:46:25 -0300 Subject: [PATCH 18/55] debug logs --- .../packages/hrm-search-config/convertToIndexDocument.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 2226cf2e2..ee3a3f684 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -236,6 +236,10 @@ const convertContactToCaseScriptUpdate = ( }, }; + console.log('>>>> documentUpdate', payload) + console.log('>>>> documentUpdate', documentUpdate) + console.log('>>>> scriptUpdate', scriptUpdate) + return { documentUpdate, scriptUpdate }; } case 'remove': { From 08e28c9c48da11717615ab9870b730ad30fb587e Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 13 May 2024 17:58:54 -0300 Subject: [PATCH 19/55] debug --- hrm-domain/lambdas/search-index-consumer/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 1ce9daa35..42fb0c4ca 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -184,6 +184,7 @@ const handleIndexPayload = console.error( new HrmIndexProcessorError('handleIndexPayload: Failed to process index request'), err, + { documentId, indexHandler, messageId, payload }, ); return { From c59db2a16189a840c3dd6f7364307b535ea825f4 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 14 May 2024 12:50:30 -0300 Subject: [PATCH 20/55] chore: factor out convertToScriptUpdate logic --- .../convertToIndexDocument.ts | 115 +----------------- .../convertToScriptUpdate.ts | 109 +++++++++++++++++ .../packages/hrm-search-config/index.ts | 10 +- .../packages/hrm-search-config/payload.ts | 56 +++++++++ 4 files changed, 171 insertions(+), 119 deletions(-) create mode 100644 hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts create mode 100644 hrm-domain/packages/hrm-search-config/payload.ts diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index ee3a3f684..e3a48207a 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,9 +23,6 @@ * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html */ -import type { Script } from '@elastic/elasticsearch/lib/api/types'; -import type { CaseService, Contact } from '@tech-matters/hrm-types'; -import { assertExhaustive, AccountSID } from '@tech-matters/types'; import { ContactDocument, CaseDocument, @@ -33,36 +30,7 @@ import { HRM_CASES_INDEX_TYPE, } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; - -type IndexOperation = 'index' | 'remove'; - -type IndexContactMessage = { - type: 'contact'; - operation: IndexOperation; - contact: Pick & Partial; -}; - -type IndexCaseMessage = { - type: 'case'; - operation: IndexOperation; - case: Pick & - Partial> & { - sections: NonNullable; - }; -}; - -export type IndexMessage = { accountSid: AccountSID } & ( - | IndexContactMessage - | IndexCaseMessage -); - -type IndexPayloadContact = IndexContactMessage & { - transcript: NonNullable; -}; - -type IndexPayloadCase = IndexCaseMessage; - -export type IndexPayload = IndexPayloadContact | IndexPayloadCase; +import { IndexPayload, IndexPayloadCase, IndexPayloadContact } from './payload'; const filterEmpty = (doc: T): T => Object.entries(doc).reduce((accum, [key, value]) => { @@ -73,7 +41,7 @@ const filterEmpty = (doc: T): T => return accum; }, {} as T); -const convertContactToContactDocument = ({ +export const convertContactToContactDocument = ({ contact, transcript, }: IndexPayloadContact): CreateIndexConvertedDocument => { @@ -208,82 +176,3 @@ export const convertToIndexDocument = ( throw new Error(`convertToIndexDocument not implemented for index ${indexName}`); }; - -const convertContactToCaseScriptUpdate = ( - payload: IndexPayloadContact, -): { - documentUpdate: CreateIndexConvertedDocument; - scriptUpdate: Script; -} => { - const { operation } = payload; - const { accountSid, caseId } = payload.contact; - - switch (operation) { - case 'index': { - const contactDocument = convertContactToContactDocument(payload); - - const documentUpdate: CreateIndexConvertedDocument = { - id: parseInt(caseId!, 10), - accountSid, - contacts: [contactDocument], - }; - - const scriptUpdate: Script = { - source: - 'def replaceContact(Map newContact, List contacts) { contacts.removeIf(contact -> contact.id == newContact.id); contacts.add(newContact); } replaceContact(params.newContact, ctx._source.contacts);', - params: { - newContact: contactDocument, - }, - }; - - console.log('>>>> documentUpdate', payload) - console.log('>>>> documentUpdate', documentUpdate) - console.log('>>>> scriptUpdate', scriptUpdate) - - return { documentUpdate, scriptUpdate }; - } - case 'remove': { - const scriptUpdate: Script = { - source: - 'def removeContact(int contactId, List contacts) { contacts.removeIf(contact -> contact.id == contactId); } removeContact(params.contactId, ctx._source.contacts);', - params: { - contactId: payload.contact.id, - }, - }; - - return { documentUpdate: {}, scriptUpdate }; - } - default: { - return assertExhaustive(operation); - } - } -}; - -const convertToCaseScriptUpdate = ( - payload: IndexPayload, -): { - documentUpdate: CreateIndexConvertedDocument; - scriptUpdate: Script; -} => { - if (payload.type === 'contact') { - return convertContactToCaseScriptUpdate(payload); - } - - throw new Error( - `convertToCaseScriptDocument not implemented for type ${payload.type} and operation ${payload.operation}`, - ); -}; - -export const convertToScriptUpdate = ( - payload: IndexPayload, - indexName: string, -): { - documentUpdate: CreateIndexConvertedDocument; - scriptUpdate: Script; -} => { - if (indexName.endsWith(HRM_CASES_INDEX_TYPE)) { - return convertToCaseScriptUpdate(payload); - } - - throw new Error(`convertToScriptDocument not implemented for index ${indexName}`); -}; diff --git a/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts b/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts new file mode 100644 index 000000000..d918d3e57 --- /dev/null +++ b/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +/** + * This is a very early example of a rudimentary configuration for a multi-language index in ES. + * + * There is a lot of room for improvement here to allow more robust use of the ES query string + * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. + * + * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ +import type { Script } from '@elastic/elasticsearch/lib/api/types'; +import { assertExhaustive } from '@tech-matters/types'; +import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; +import { IndexPayload, IndexPayloadContact } from './payload'; +import { + CaseDocument, + ContactDocument, + HRM_CASES_INDEX_TYPE, +} from './hrmIndexDocumentMappings'; +import { convertContactToContactDocument } from './convertToIndexDocument'; + +const convertContactToCaseScriptUpdate = ( + payload: IndexPayloadContact, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + const { operation } = payload; + const { accountSid, caseId } = payload.contact; + + switch (operation) { + case 'index': { + const contactDocument = convertContactToContactDocument(payload); + + const documentUpdate: CreateIndexConvertedDocument = { + id: parseInt(caseId!, 10), + accountSid, + contacts: [contactDocument], + }; + + const scriptUpdate: Script = { + source: + 'def replaceContact(Map newContact, List contacts) { contacts.removeIf(contact -> contact.id == newContact.id); contacts.add(newContact); } replaceContact(params.newContact, ctx._source.contacts);', + params: { + newContact: contactDocument, + }, + }; + + return { documentUpdate, scriptUpdate }; + } + case 'remove': { + const scriptUpdate: Script = { + source: + 'def removeContact(int contactId, List contacts) { contacts.removeIf(contact -> contact.id == contactId); } removeContact(params.contactId, ctx._source.contacts);', + params: { + contactId: payload.contact.id, + }, + }; + + return { documentUpdate: {}, scriptUpdate }; + } + default: { + return assertExhaustive(operation); + } + } +}; + +const convertToCaseScriptUpdate = ( + payload: IndexPayload, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + if (payload.type === 'contact') { + return convertContactToCaseScriptUpdate(payload); + } + + throw new Error( + `convertToCaseScriptDocument not implemented for type ${payload.type} and operation ${payload.operation}`, + ); +}; + +export const convertToScriptUpdate = ( + payload: IndexPayload, + indexName: string, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + if (indexName.endsWith(HRM_CASES_INDEX_TYPE)) { + return convertToCaseScriptUpdate(payload); + } + + throw new Error(`convertToScriptDocument not implemented for index ${indexName}`); +}; diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index c4c76df56..10017f9dc 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -14,22 +14,20 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { - IndexPayload, - convertToIndexDocument, - convertToScriptUpdate, -} from './convertToIndexDocument'; +import { convertToIndexDocument } from './convertToIndexDocument'; +import { convertToScriptUpdate } from './convertToScriptUpdate'; import { getCreateIndexParams } from './getCreateIndexParams'; import type { IndexConfiguration, SearchConfiguration, } from '@tech-matters/elasticsearch-client'; +import { IndexPayload } from './payload'; export { HRM_CASES_INDEX_TYPE, HRM_CONTACTS_INDEX_TYPE, } from './hrmIndexDocumentMappings'; -export { IndexMessage, IndexPayload } from './convertToIndexDocument'; +export { IndexMessage, IndexPayload } from './payload'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { diff --git a/hrm-domain/packages/hrm-search-config/payload.ts b/hrm-domain/packages/hrm-search-config/payload.ts new file mode 100644 index 000000000..2627cab9e --- /dev/null +++ b/hrm-domain/packages/hrm-search-config/payload.ts @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +/** + * This is a very early example of a rudimentary configuration for a multi-language index in ES. + * + * There is a lot of room for improvement here to allow more robust use of the ES query string + * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. + * + * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ +import type { CaseService, Contact } from '@tech-matters/hrm-types'; +import { AccountSID } from '@tech-matters/types'; + +type IndexOperation = 'index' | 'remove'; + +type IndexContactMessage = { + type: 'contact'; + operation: IndexOperation; + contact: Pick & Partial; +}; + +type IndexCaseMessage = { + type: 'case'; + operation: IndexOperation; + case: Pick & + Partial> & { + sections: NonNullable; + }; +}; + +export type IndexMessage = { accountSid: AccountSID } & ( + | IndexContactMessage + | IndexCaseMessage +); + +export type IndexPayloadContact = IndexContactMessage & { + transcript: NonNullable; +}; + +export type IndexPayloadCase = IndexCaseMessage; + +export type IndexPayload = IndexPayloadContact | IndexPayloadCase; From 4101c0c211612b42f75c738703ad41ab572b93a1 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 14 May 2024 16:51:51 -0300 Subject: [PATCH 21/55] chore: refactor search-index-consumer to improve redability --- .../lambdas/search-index-consumer/index.ts | 238 +----------------- .../lambdas/search-index-consumer/messages.ts | 41 +++ .../messagesToPayloads.ts | 126 ++++++++++ .../search-index-consumer/payloadToIndex.ts | 138 ++++++++++ 4 files changed, 318 insertions(+), 225 deletions(-) create mode 100644 hrm-domain/lambdas/search-index-consumer/messages.ts create mode 100644 hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts create mode 100644 hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 42fb0c4ca..443fc8c07 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -13,245 +13,32 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ -import type { SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda'; -import { getClient, IndexClient } from '@tech-matters/elasticsearch-client'; +import type { SQSBatchResponse, SQSEvent } from 'aws-lambda'; // import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; import { HrmIndexProcessorError } from '@tech-matters/job-errors'; -import { - HRM_CASES_INDEX_TYPE, - HRM_CONTACTS_INDEX_TYPE, - hrmIndexConfiguration, - IndexMessage, - IndexPayload, -} from '@tech-matters/hrm-search-config'; -import { - AccountSID, - assertExhaustive, - isErr, - newErr, - newOkFromData, -} from '@tech-matters/types'; - -type MessageWithMeta = { message: IndexMessage; messageId: string }; -type MessagesByAccountSid = Record; -type PayloadWithMeta = { - documentId: number; - payload: IndexPayload; - messageId: string; - indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript'; -}; -type PayloadsByIndex = { - [indexType: string]: PayloadWithMeta[]; -}; -type PayloadsByAccountSid = Record; - -const groupMessagesByAccountSid = ( - accum: MessagesByAccountSid, - curr: SQSRecord, -): MessagesByAccountSid => { - const { messageId, body } = curr; - const message = JSON.parse(body) as IndexMessage; - - const { accountSid } = message; - - if (!accum[accountSid]) { - return { ...accum, [accountSid]: [{ messageId, message }] }; - } - - return { ...accum, [accountSid]: [...accum[accountSid], { messageId, message }] }; -}; - -const messagesToPayloadsByIndex = ( - accum: PayloadsByIndex, - currM: MessageWithMeta, -): PayloadsByIndex => { - const { message } = currM; - - const { type } = message; - - switch (type) { - case 'contact': { - // TODO: Pull the transcripts from S3 (if any) - return { - ...accum, - // add an upsert job to HRM_CONTACTS_INDEX_TYPE index - [HRM_CONTACTS_INDEX_TYPE]: [ - ...(accum[HRM_CONTACTS_INDEX_TYPE] ?? []), - { - ...currM, - documentId: message.contact.id, - payload: { ...message, transcript: '' }, - indexHandler: 'updateDocument', - }, - ], - // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: message.contact.caseId - ? [ - ...(accum[HRM_CASES_INDEX_TYPE] ?? []), - { - ...currM, - documentId: parseInt(message.contact.caseId, 10), - payload: { ...message, transcript: '' }, - indexHandler: 'updateScript', - }, - ] - : accum[HRM_CASES_INDEX_TYPE] ?? [], - }; - } - case 'case': { - return { - ...accum, - // add an upsert job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: [ - ...(accum[HRM_CASES_INDEX_TYPE] ?? []), - { - ...currM, - documentId: message.case.id, - payload: { ...message }, - indexHandler: 'updateDocument', - }, - ], - }; - } - default: { - return assertExhaustive(type); - } - } -}; - -const handleIndexPayload = - ({ - accountSid, - client, - indexType, - }: { - accountSid: string; - client: IndexClient; - indexType: string; - }) => - async ({ documentId, indexHandler, messageId, payload }: PayloadWithMeta) => { - try { - switch (indexHandler) { - case 'indexDocument': { - const result = await client.indexDocument({ - id: documentId.toString(), - document: payload, - autocreate: true, - }); - - return { - accountSid, - indexType, - messageId, - result: newOkFromData(result), - }; - } - case 'updateDocument': { - const result = await client.updateDocument({ - id: documentId.toString(), - document: payload, - autocreate: true, - docAsUpsert: true, - }); - - return { - accountSid, - indexType, - messageId, - result: newOkFromData(result), - }; - } - case 'updateScript': { - const result = await client.updateScript({ - document: payload, - id: documentId.toString(), - autocreate: true, - scriptedUpsert: true, - }); - - return { - accountSid, - indexType, - messageId, - result: newOkFromData(result), - }; - } - default: { - return assertExhaustive(indexHandler); - } - } - } catch (err) { - console.error( - new HrmIndexProcessorError('handleIndexPayload: Failed to process index request'), - err, - { documentId, indexHandler, messageId, payload }, - ); - - return { - accountSid, - indexType, - messageId, - result: newErr({ - error: 'ErrorFailedToInex', - message: err instanceof Error ? err.message : String(err), - }), - }; - } - }; - -const indexDocumentsByIndex = - (accountSid: string) => - async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { - // get the client for the accountSid-indexType pair - const client = (await getClient({ accountSid, indexType })).indexClient( - hrmIndexConfiguration, - ); - - const mapper = handleIndexPayload({ client, accountSid, indexType }); - - const indexed = await Promise.all(payloads.map(mapper)); - - return indexed; - }; - -const indexDocumentsByAccount = async ([accountSid, payloadsByIndex]: [ - string, - PayloadsByIndex, -]) => { - const resultsByIndex = await Promise.all( - Object.entries(payloadsByIndex).map(indexDocumentsByIndex(accountSid)), - ); - - return resultsByIndex; -}; +import { isErr } from '@tech-matters/types'; +import { groupMessagesByAccountSid } from './messages'; +import { messagesToPayloadsByAccountSid } from './messagesToPayloads'; +import { indexDocumentsByAccount } from './payloadToIndex'; export const handler = async (event: SQSEvent): Promise => { console.debug('Received event:', JSON.stringify(event, null, 2)); try { // group the messages by accountSid while adding message meta - const messagesByAccoundSid = event.Records.reduce( - groupMessagesByAccountSid, - {}, - ); + const messagesByAccoundSid = groupMessagesByAccountSid(event.Records); - // generate corresponding IndexPayload for each IndexMessage and group them by target indexType - const documentsByAccountSid: PayloadsByAccountSid = Object.fromEntries( - Object.entries(messagesByAccoundSid).map(([accountSid, messages]) => { - const payloads = messages.reduce(messagesToPayloadsByIndex, {}); + // generate corresponding IndexPayload for each IndexMessage and group them by target accountSid-indexType pair + const payloadsByAccountSid = messagesToPayloadsByAccountSid(messagesByAccoundSid); - return [accountSid, payloads] as const; - }), - ); + console.debug('Mapped messages:', JSON.stringify(payloadsByAccountSid, null, 2)); - console.debug('Mapped messages:', JSON.stringify(documentsByAccountSid, null, 2)); - - const resultsByAccount = await Promise.all( - Object.entries(documentsByAccountSid).map(indexDocumentsByAccount), - ); + // index all the payloads + const resultsByAccount = await indexDocumentsByAccount(payloadsByAccountSid); console.debug(`Successfully indexed documents`); + // filter the payloads that failed indexing const documentsWithErrors = resultsByAccount .flat(2) .filter(({ result }) => isErr(result)); @@ -263,6 +50,7 @@ export const handler = async (event: SQSEvent): Promise => { ); } + // send the failed payloads back to SQS so they are redrive to DLQ const response: SQSBatchResponse = { batchItemFailures: documentsWithErrors.map(({ messageId }) => ({ itemIdentifier: messageId, diff --git a/hrm-domain/lambdas/search-index-consumer/messages.ts b/hrm-domain/lambdas/search-index-consumer/messages.ts new file mode 100644 index 000000000..447fd01fd --- /dev/null +++ b/hrm-domain/lambdas/search-index-consumer/messages.ts @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import type { IndexMessage } from '@tech-matters/hrm-search-config'; +import type { AccountSID } from '@tech-matters/types'; +import type { SQSRecord } from 'aws-lambda'; + +export type MessageWithMeta = { message: IndexMessage; messageId: string }; +export type MessagesByAccountSid = Record; + +const groupMessagesReducer = ( + accum: MessagesByAccountSid, + curr: SQSRecord, +): MessagesByAccountSid => { + const { messageId, body } = curr; + const message = JSON.parse(body) as IndexMessage; + + const { accountSid } = message; + + if (!accum[accountSid]) { + return { ...accum, [accountSid]: [{ messageId, message }] }; + } + + return { ...accum, [accountSid]: [...accum[accountSid], { messageId, message }] }; +}; + +export const groupMessagesByAccountSid = (records: SQSRecord[]): MessagesByAccountSid => + records.reduce(groupMessagesReducer, {}); diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts new file mode 100644 index 000000000..868c15dc0 --- /dev/null +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -0,0 +1,126 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { + HRM_CASES_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, + type IndexPayload, +} from '@tech-matters/hrm-search-config'; +import { assertExhaustive, type AccountSID } from '@tech-matters/types'; +import type { MessageWithMeta, MessagesByAccountSid } from './messages'; + +/** + * A payload is single object that should be indexed in a particular index. A single message might represent multiple payloads. + * The indexHandler represents the operation that should be used when indexing the given document: + * - indexDocument: Used when we don't care overriding the previous versions of the document. An example is when a document is created for the first time + * - updateDocument: Used when we want to preserve the existing document (if any), using the document object for the update. An example is update a case + * - updateScript: Used when we want to preserve the existing document (if any), using the generated "update script" for the update. An example is updating case.contacts list when a contact is indexed + */ +export type PayloadWithMeta = { + documentId: number; + payload: IndexPayload; + messageId: string; + indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript'; +}; +export type PayloadsByIndex = { + [indexType: string]: PayloadWithMeta[]; +}; +export type PayloadsByAccountSid = Record; + +// TODO: Pull the transcripts from S3 (if any) +const generatePayloadFromContact = ( + ps: PayloadsByIndex, + m: MessageWithMeta & { message: { type: 'contact' } }, +): PayloadsByIndex => ({ + ...ps, + // add an upsert job to HRM_CONTACTS_INDEX_TYPE index + [HRM_CONTACTS_INDEX_TYPE]: [ + ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.contact.id, + payload: { ...m.message, transcript: '' }, + indexHandler: 'updateDocument', + }, + ], + // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId + ? [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: parseInt(m.message.contact.caseId, 10), + payload: { ...m.message, transcript: '' }, + indexHandler: 'updateScript', + }, + ] + : ps[HRM_CASES_INDEX_TYPE] ?? [], +}); + +const generatePayloadFromCase = ( + ps: PayloadsByIndex, + m: MessageWithMeta & { message: { type: 'case' } }, +): PayloadsByIndex => ({ + ...ps, + // add an upsert job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.case.id, + payload: { ...m.message }, + indexHandler: 'updateDocument', + }, + ], +}); + +const messagesToPayloadReducer = ( + accum: PayloadsByIndex, + currM: MessageWithMeta, +): PayloadsByIndex => { + const { message, messageId } = currM; + + const { type } = message; + + switch (type) { + case 'contact': { + return generatePayloadFromContact(accum, { message, messageId }); + } + case 'case': { + return generatePayloadFromCase(accum, { message, messageId }); + } + default: { + return assertExhaustive(type); + } + } +}; + +const messagesToPayloadsByIndex = (messages: MessageWithMeta[]): PayloadsByIndex => + messages.reduce(messagesToPayloadReducer, {}); + +export const messagesToPayloadsByAccountSid = ( + messages: MessagesByAccountSid, +): PayloadsByAccountSid => { + const payloadsByAccountSidEntries = Object.entries(messages).map(([accountSid, ms]) => { + const payloads = messagesToPayloadsByIndex(ms); + + return [accountSid, payloads] as const; + }); + + const payloadsByAccountSid = Object.fromEntries(payloadsByAccountSidEntries); + + return payloadsByAccountSid; +}; diff --git a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts new file mode 100644 index 000000000..0e6b96d29 --- /dev/null +++ b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts @@ -0,0 +1,138 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { type IndexClient, getClient } from '@tech-matters/elasticsearch-client'; +import { + type IndexPayload, + hrmIndexConfiguration, +} from '@tech-matters/hrm-search-config'; +import type { + PayloadWithMeta, + PayloadsByAccountSid, + PayloadsByIndex, +} from './messagesToPayloads'; +import { assertExhaustive, newErr, newOkFromData } from '@tech-matters/types'; +import { HrmIndexProcessorError } from '@tech-matters/job-errors'; + +const handleIndexPayload = + ({ + accountSid, + client, + indexType, + }: { + accountSid: string; + client: IndexClient; + indexType: string; + }) => + async ({ documentId, indexHandler, messageId, payload }: PayloadWithMeta) => { + try { + switch (indexHandler) { + case 'indexDocument': { + const result = await client.indexDocument({ + id: documentId.toString(), + document: payload, + autocreate: true, + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + case 'updateDocument': { + const result = await client.updateDocument({ + id: documentId.toString(), + document: payload, + autocreate: true, + docAsUpsert: true, + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + case 'updateScript': { + const result = await client.updateScript({ + document: payload, + id: documentId.toString(), + autocreate: true, + scriptedUpsert: true, + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + default: { + return assertExhaustive(indexHandler); + } + } + } catch (err) { + console.error( + new HrmIndexProcessorError('handleIndexPayload: Failed to process index request'), + err, + { documentId, indexHandler, messageId, payload }, + ); + + return { + accountSid, + indexType, + messageId, + result: newErr({ + error: 'ErrorFailedToInex', + message: err instanceof Error ? err.message : String(err), + }), + }; + } + }; + +const indexDocumentsByIndexMapper = + (accountSid: string) => + async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { + // get the client for the accountSid-indexType pair + const client = (await getClient({ accountSid, indexType })).indexClient( + hrmIndexConfiguration, + ); + + const mapper = handleIndexPayload({ client, accountSid, indexType }); + + const indexed = await Promise.all(payloads.map(mapper)); + + return indexed; + }; + +const indexDocumentsByAccountMapper = async ([accountSid, payloadsByIndex]: [ + string, + PayloadsByIndex, +]) => { + const resultsByIndex = await Promise.all( + Object.entries(payloadsByIndex).map(indexDocumentsByIndexMapper(accountSid)), + ); + + return resultsByIndex; +}; + +export const indexDocumentsByAccount = async ( + payloadsByAccountSid: PayloadsByAccountSid, +) => Promise.all(Object.entries(payloadsByAccountSid).map(indexDocumentsByAccountMapper)); From 7df50bd907d6a617f23905f9d8db9edf54749b48 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 14 May 2024 17:49:33 -0300 Subject: [PATCH 22/55] chore: lint --- .../convertToIndexDocument.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index e3a48207a..1ea4ce253 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -65,16 +65,16 @@ export const convertContactToContactDocument = ({ accountSid, id, createdAt, - updatedAt: updatedAt, - createdBy: createdBy, - updatedBy: updatedBy, + updatedAt, + createdBy, + updatedBy, finalized: Boolean(finalizedAt), - helpline: helpline, - channel: channel, - number: number, - timeOfContact: timeOfContact, + helpline, + channel, + number, + timeOfContact, transcript, - twilioWorkerId: twilioWorkerId, + twilioWorkerId, content: JSON.stringify(rawJson), // high_boost_global: '', // highBoostGlobal.join(' '), // low_boost_global: '', // lowBoostGlobal.join(' '), @@ -129,9 +129,9 @@ const convertCaseToCaseDocument = ({ helpline, twilioWorkerId, status, - previousStatus: previousStatus, - statusUpdatedAt: statusUpdatedAt, - statusUpdatedBy: statusUpdatedBy, + previousStatus, + statusUpdatedAt, + statusUpdatedBy, content: JSON.stringify(info), sections: mappedSections, contacts: null, From 06f0254b0c9e154bcb2bf1184d543a0d8699b1fe Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 14 May 2024 18:08:52 -0300 Subject: [PATCH 23/55] chore: removed comments --- .../packages/hrm-search-config/convertToIndexDocument.ts | 9 --------- .../packages/hrm-search-config/convertToScriptUpdate.ts | 9 --------- .../packages/hrm-search-config/getCreateIndexParams.ts | 9 --------- hrm-domain/packages/hrm-search-config/payload.ts | 8 -------- 4 files changed, 35 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 1ea4ce253..f1ae36ced 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -14,15 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -/** - * This is a very early example of a rudimentary configuration for a multi-language index in ES. - * - * There is a lot of room for improvement here to allow more robust use of the ES query string - * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. - * - * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - */ - import { ContactDocument, CaseDocument, diff --git a/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts b/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts index d918d3e57..23ac05389 100644 --- a/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts +++ b/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts @@ -13,15 +13,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ - -/** - * This is a very early example of a rudimentary configuration for a multi-language index in ES. - * - * There is a lot of room for improvement here to allow more robust use of the ES query string - * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. - * - * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - */ import type { Script } from '@elastic/elasticsearch/lib/api/types'; import { assertExhaustive } from '@tech-matters/types'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; diff --git a/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts b/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts index 5850c6cd1..884ce917a 100644 --- a/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts +++ b/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts @@ -14,15 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -/** - * This is a very early example of a rudimentary configuration for a multi-language index in ES. - * - * There is a lot of room for improvement here to allow more robust use of the ES query string - * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. - * - * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - */ - import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types'; import { diff --git a/hrm-domain/packages/hrm-search-config/payload.ts b/hrm-domain/packages/hrm-search-config/payload.ts index 2627cab9e..379976a2e 100644 --- a/hrm-domain/packages/hrm-search-config/payload.ts +++ b/hrm-domain/packages/hrm-search-config/payload.ts @@ -14,14 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -/** - * This is a very early example of a rudimentary configuration for a multi-language index in ES. - * - * There is a lot of room for improvement here to allow more robust use of the ES query string - * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. - * - * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - */ import type { CaseService, Contact } from '@tech-matters/hrm-types'; import { AccountSID } from '@tech-matters/types'; From b208991c1343d77c852bae211302d03258c50a49 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 16 May 2024 16:00:38 -0300 Subject: [PATCH 24/55] chore: moved channelTypes constants to hrm-types module --- hrm-domain/hrm-core/contact/contactService.ts | 2 +- hrm-domain/hrm-service/package.json | 1 + .../contact-job/jobTypes/retrieveTranscript.test.ts | 2 +- .../service-tests/contact/contactConversationMedia.test.ts | 2 +- .../{hrm-core/contact => packages/hrm-types}/channelTypes.ts | 1 - hrm-domain/packages/hrm-types/index.ts | 1 + package-lock.json | 2 ++ 7 files changed, 7 insertions(+), 4 deletions(-) rename hrm-domain/{hrm-core/contact => packages/hrm-types}/channelTypes.ts (95%) diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index 7435ed9c0..655631639 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -50,7 +50,7 @@ import { actionsMaps } from '../permissions'; import type { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { createReferral } from '../referral/referral-model'; import { createContactJob } from '../contact-job/contact-job'; -import { isChatChannel } from './channelTypes'; +import { isChatChannel } from '@tech-matters/hrm-types'; import { enableCreateContactJobsFlag } from '../featureFlags'; import { db } from '../connection-pool'; import { diff --git a/hrm-domain/hrm-service/package.json b/hrm-domain/hrm-service/package.json index cbbefe1bc..770d9ee79 100644 --- a/hrm-domain/hrm-service/package.json +++ b/hrm-domain/hrm-service/package.json @@ -50,6 +50,7 @@ "@tech-matters/contact-job-cleanup": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/hrm-data-pull": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/profile-flags-cleanup": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", diff --git a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts index e330789ee..69d0af6b8 100644 --- a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts @@ -23,7 +23,7 @@ import * as contactJobApi from '@tech-matters/hrm-core/contact-job/contact-job-d import { db } from '@tech-matters/hrm-core/connection-pool'; import '../../case/caseValidation'; import * as conversationMediaApi from '@tech-matters/hrm-core/conversation-media/conversation-media'; -import { chatChannels } from '@tech-matters/hrm-core/contact/channelTypes'; +import { chatChannels } from '@tech-matters/hrm-types'; import { JOB_MAX_ATTEMPTS } from '@tech-matters/hrm-core/contact-job/contact-job-processor'; import { diff --git a/hrm-domain/hrm-service/service-tests/contact/contactConversationMedia.test.ts b/hrm-domain/hrm-service/service-tests/contact/contactConversationMedia.test.ts index 4f0d73c17..a6bf4a4ba 100644 --- a/hrm-domain/hrm-service/service-tests/contact/contactConversationMedia.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact/contactConversationMedia.test.ts @@ -34,7 +34,7 @@ import { cleanupReferrals, } from './dbCleanup'; import each from 'jest-each'; -import { chatChannels } from '@tech-matters/hrm-core/contact/channelTypes'; +import { chatChannels } from '@tech-matters/hrm-types'; import { ContactJobType } from '@tech-matters/types/ContactJob'; import { ruleFileActionOverride } from '../permissions-overrides'; import { selectJobsByContactId } from './db-validations'; diff --git a/hrm-domain/hrm-core/contact/channelTypes.ts b/hrm-domain/packages/hrm-types/channelTypes.ts similarity index 95% rename from hrm-domain/hrm-core/contact/channelTypes.ts rename to hrm-domain/packages/hrm-types/channelTypes.ts index e940822df..8d97d64de 100644 --- a/hrm-domain/hrm-core/contact/channelTypes.ts +++ b/hrm-domain/packages/hrm-types/channelTypes.ts @@ -40,6 +40,5 @@ export const chatChannels = [ channelTypes.modica, ]; -// eslint-disable-next-line @typescript-eslint/no-unused-vars export const isVoiceChannel = (channel: string) => channel === channelTypes.voice; export const isChatChannel = (channel: string) => chatChannels.includes(channel as any); diff --git a/hrm-domain/packages/hrm-types/index.ts b/hrm-domain/packages/hrm-types/index.ts index 8366a362a..f2bd9fedd 100644 --- a/hrm-domain/packages/hrm-types/index.ts +++ b/hrm-domain/packages/hrm-types/index.ts @@ -19,3 +19,4 @@ export * from './Referral'; export * from './ConversationMedia'; export * from './Case'; export * from './CaseSection'; +export * from './channelTypes'; diff --git a/package-lock.json b/package-lock.json index 1d3847b6b..4d68813c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -142,6 +142,7 @@ "@tech-matters/contact-job-cleanup": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/hrm-data-pull": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/profile-flags-cleanup": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", @@ -17644,6 +17645,7 @@ "@tech-matters/contact-job-cleanup": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/hrm-data-pull": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/profile-flags-cleanup": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", From acb12451a99d0f5fb51209f6863c849a2da07d91 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 16 May 2024 18:39:08 -0300 Subject: [PATCH 25/55] chore fetch transcripts from S3 --- .../lambdas/search-index-consumer/index.ts | 4 +- .../messagesToPayloads.ts | 152 +++++++++++++----- .../packages/hrm-search-config/index.ts | 7 +- .../packages/hrm-search-config/payload.ts | 6 +- 4 files changed, 124 insertions(+), 45 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 443fc8c07..576343531 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -14,7 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ import type { SQSBatchResponse, SQSEvent } from 'aws-lambda'; -// import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; import { HrmIndexProcessorError } from '@tech-matters/job-errors'; import { isErr } from '@tech-matters/types'; import { groupMessagesByAccountSid } from './messages'; @@ -29,7 +28,8 @@ export const handler = async (event: SQSEvent): Promise => { const messagesByAccoundSid = groupMessagesByAccountSid(event.Records); // generate corresponding IndexPayload for each IndexMessage and group them by target accountSid-indexType pair - const payloadsByAccountSid = messagesToPayloadsByAccountSid(messagesByAccoundSid); + const payloadsByAccountSid = + await messagesToPayloadsByAccountSid(messagesByAccoundSid); console.debug('Mapped messages:', JSON.stringify(payloadsByAccountSid, null, 2)); diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts index 868c15dc0..123063f44 100644 --- a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -13,13 +13,16 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ - +import { getS3Object } from '@tech-matters/s3-client'; import { HRM_CASES_INDEX_TYPE, HRM_CONTACTS_INDEX_TYPE, type IndexPayload, + type IndexContactMessage, + type IndexCaseMessage, } from '@tech-matters/hrm-search-config'; import { assertExhaustive, type AccountSID } from '@tech-matters/types'; +import { isChatChannel, isS3StoredTranscript } from '@tech-matters/hrm-types'; import type { MessageWithMeta, MessagesByAccountSid } from './messages'; /** @@ -40,39 +43,100 @@ export type PayloadsByIndex = { }; export type PayloadsByAccountSid = Record; -// TODO: Pull the transcripts from S3 (if any) +type IntermidiateIndexContactMessage = MessageWithMeta & { + message: IndexContactMessage; +} & { + transcript: string | null; +}; + +const intermediateContactMessage = async ( + m: MessageWithMeta & { + message: IndexContactMessage; + }, +): Promise => { + let transcript: string | null = null; + + if (m.message.contact.channel && isChatChannel(m.message.contact.channel)) { + const transcriptEntry = + m.message.contact.conversationMedia?.find(isS3StoredTranscript); + if (transcriptEntry) { + const { location } = transcriptEntry.storeTypeSpecificData; + const { bucket, key } = location || {}; + if (bucket && key) { + transcript = await getS3Object({ bucket, key }); + } + } + } + + return { ...m, transcript }; +}; + +type IntermidiateIndexCaseMessage = MessageWithMeta & { + message: IndexCaseMessage; +}; +const intermediateCaseMessage = async ( + m: MessageWithMeta & { + message: IndexCaseMessage; + }, +): Promise => m; + +type IntermidiateIndexMessage = + | IntermidiateIndexContactMessage + | IntermidiateIndexCaseMessage; +const intermediateMessagesMapper = ( + m: MessageWithMeta, +): Promise => { + const { message, messageId } = m; + + const { type } = message; + + switch (type) { + case 'contact': { + return intermediateContactMessage({ message, messageId }); + } + case 'case': { + return intermediateCaseMessage({ message, messageId }); + } + default: { + return assertExhaustive(type); + } + } +}; + const generatePayloadFromContact = ( ps: PayloadsByIndex, - m: MessageWithMeta & { message: { type: 'contact' } }, -): PayloadsByIndex => ({ - ...ps, - // add an upsert job to HRM_CONTACTS_INDEX_TYPE index - [HRM_CONTACTS_INDEX_TYPE]: [ - ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), - { - ...m, - documentId: m.message.contact.id, - payload: { ...m.message, transcript: '' }, - indexHandler: 'updateDocument', - }, - ], - // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId - ? [ - ...(ps[HRM_CASES_INDEX_TYPE] ?? []), - { - ...m, - documentId: parseInt(m.message.contact.caseId, 10), - payload: { ...m.message, transcript: '' }, - indexHandler: 'updateScript', - }, - ] - : ps[HRM_CASES_INDEX_TYPE] ?? [], -}); + m: IntermidiateIndexContactMessage, +): PayloadsByIndex => { + return { + ...ps, + // add an upsert job to HRM_CONTACTS_INDEX_TYPE index + [HRM_CONTACTS_INDEX_TYPE]: [ + ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.contact.id, + payload: { ...m.message, transcript: m.transcript }, + indexHandler: 'updateDocument', + }, + ], + // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId + ? [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: parseInt(m.message.contact.caseId, 10), + payload: { ...m.message, transcript: m.transcript }, + indexHandler: 'updateScript', + }, + ] + : ps[HRM_CASES_INDEX_TYPE] ?? [], + }; +}; const generatePayloadFromCase = ( ps: PayloadsByIndex, - m: MessageWithMeta & { message: { type: 'case' } }, + m: IntermidiateIndexCaseMessage, ): PayloadsByIndex => ({ ...ps, // add an upsert job to HRM_CASES_INDEX_TYPE index @@ -89,7 +153,7 @@ const generatePayloadFromCase = ( const messagesToPayloadReducer = ( accum: PayloadsByIndex, - currM: MessageWithMeta, + currM: IntermidiateIndexMessage, ): PayloadsByIndex => { const { message, messageId } = currM; @@ -97,7 +161,8 @@ const messagesToPayloadReducer = ( switch (type) { case 'contact': { - return generatePayloadFromContact(accum, { message, messageId }); + const { transcript } = currM as IntermidiateIndexContactMessage; + return generatePayloadFromContact(accum, { message, messageId, transcript }); } case 'case': { return generatePayloadFromCase(accum, { message, messageId }); @@ -108,17 +173,26 @@ const messagesToPayloadReducer = ( } }; -const messagesToPayloadsByIndex = (messages: MessageWithMeta[]): PayloadsByIndex => - messages.reduce(messagesToPayloadReducer, {}); +const messagesToPayloadsByIndex = async ( + messages: MessageWithMeta[], +): Promise => { + const intermidiateMessages = await Promise.all( + messages.map(intermediateMessagesMapper), + ); + + return intermidiateMessages.reduce(messagesToPayloadReducer, {}); +}; -export const messagesToPayloadsByAccountSid = ( +export const messagesToPayloadsByAccountSid = async ( messages: MessagesByAccountSid, -): PayloadsByAccountSid => { - const payloadsByAccountSidEntries = Object.entries(messages).map(([accountSid, ms]) => { - const payloads = messagesToPayloadsByIndex(ms); +): Promise => { + const payloadsByAccountSidEntries = await Promise.all( + Object.entries(messages).map(async ([accountSid, ms]) => { + const payloads = await messagesToPayloadsByIndex(ms); - return [accountSid, payloads] as const; - }); + return [accountSid, payloads] as const; + }), + ); const payloadsByAccountSid = Object.fromEntries(payloadsByAccountSidEntries); diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index 10017f9dc..352a2105e 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -27,7 +27,12 @@ export { HRM_CASES_INDEX_TYPE, HRM_CONTACTS_INDEX_TYPE, } from './hrmIndexDocumentMappings'; -export { IndexMessage, IndexPayload } from './payload'; +export { + IndexMessage, + IndexCaseMessage, + IndexContactMessage, + IndexPayload, +} from './payload'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { diff --git a/hrm-domain/packages/hrm-search-config/payload.ts b/hrm-domain/packages/hrm-search-config/payload.ts index 379976a2e..dbca12692 100644 --- a/hrm-domain/packages/hrm-search-config/payload.ts +++ b/hrm-domain/packages/hrm-search-config/payload.ts @@ -19,13 +19,13 @@ import { AccountSID } from '@tech-matters/types'; type IndexOperation = 'index' | 'remove'; -type IndexContactMessage = { +export type IndexContactMessage = { type: 'contact'; operation: IndexOperation; contact: Pick & Partial; }; -type IndexCaseMessage = { +export type IndexCaseMessage = { type: 'case'; operation: IndexOperation; case: Pick & @@ -40,7 +40,7 @@ export type IndexMessage = { accountSid: AccountSID } & ( ); export type IndexPayloadContact = IndexContactMessage & { - transcript: NonNullable; + transcript: string | null; }; export type IndexPayloadCase = IndexCaseMessage; From 0982e55599a38a77635cf489aa35df9daac72c75 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 16 May 2024 18:41:53 -0300 Subject: [PATCH 26/55] debug --- .../lambdas/search-index-consumer/messagesToPayloads.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts index 123063f44..063ebe4a5 100644 --- a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -59,11 +59,15 @@ const intermediateContactMessage = async ( if (m.message.contact.channel && isChatChannel(m.message.contact.channel)) { const transcriptEntry = m.message.contact.conversationMedia?.find(isS3StoredTranscript); + + console.log('>>>>>>>> transcriptEntry', transcriptEntry); + if (transcriptEntry) { const { location } = transcriptEntry.storeTypeSpecificData; const { bucket, key } = location || {}; if (bucket && key) { transcript = await getS3Object({ bucket, key }); + console.log('>>>>>>>> transcript', transcript); } } } From 9ef239b0313542505ad0437f152e7bc75d475599 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 16 May 2024 19:04:22 -0300 Subject: [PATCH 27/55] chore: add packages/s3-client to tsconfig.build --- hrm-domain/lambdas/search-index-consumer/tsconfig.build.json | 1 + 1 file changed, 1 insertion(+) diff --git a/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json b/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json index a8f8b404b..22ca951e5 100644 --- a/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json +++ b/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json @@ -5,6 +5,7 @@ "references": [ { "path": "packages/types" }, { "path": "packages/job-errors" }, + { "path": "packages/s3-client" }, { "path": "packages/ssm-cache" }, { "path": "packages/elasticsearch-client" }, { "path": "hrm-domain/packages/hrm-types" }, From c0a62d16489627aba794fad1489e540bc315f6aa Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 17 May 2024 13:32:39 -0300 Subject: [PATCH 28/55] chore: remove debug logs --- hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts index 063ebe4a5..521567975 100644 --- a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -60,14 +60,11 @@ const intermediateContactMessage = async ( const transcriptEntry = m.message.contact.conversationMedia?.find(isS3StoredTranscript); - console.log('>>>>>>>> transcriptEntry', transcriptEntry); - if (transcriptEntry) { const { location } = transcriptEntry.storeTypeSpecificData; const { bucket, key } = location || {}; if (bucket && key) { transcript = await getS3Object({ bucket, key }); - console.log('>>>>>>>> transcript', transcript); } } } From 838780423141202fa94af44b1a91934bd34e1fe2 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 21 May 2024 12:35:18 -0300 Subject: [PATCH 29/55] chore: add support for parameterized ES config --- packages/elasticsearch-client/src/client.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/elasticsearch-client/src/client.ts b/packages/elasticsearch-client/src/client.ts index 0286d781d..265d76502 100644 --- a/packages/elasticsearch-client/src/client.ts +++ b/packages/elasticsearch-client/src/client.ts @@ -50,6 +50,7 @@ type AccountSidOrShortCodeRequired = export type GetClientArgs = { config?: ClientOptions; indexType: string; + ssmConfigParameter?: string; } & AccountSidOrShortCodeRequired; export type GetClientOrMockArgs = GetClientArgs & { @@ -69,9 +70,11 @@ const getConfigSsmParameterKey = (indexType: string) => const getEsConfig = async ({ config, indexType, + ssmConfigParameter, }: { config: ClientOptions | undefined; indexType: string; + ssmConfigParameter?: string; }) => { console.log('config', config); if (config) return config; @@ -91,6 +94,10 @@ const getEsConfig = async ({ }; } + if (ssmConfigParameter) { + return JSON.parse(await getSsmParameter(ssmConfigParameter)); + } + return JSON.parse(await getSsmParameter(getConfigSsmParameterKey(indexType))); }; @@ -107,14 +114,21 @@ export type IndexClient = { deleteIndex: () => Promise; }; -const getClientOrMock = async ({ config, index, indexType }: GetClientOrMockArgs) => { +const getClientOrMock = async ({ + config, + index, + indexType, + ssmConfigParameter, +}: GetClientOrMockArgs) => { // TODO: mock client for unit tests // if (authToken === 'mockAuthToken') { // const mock = (getMockClient({ config }) as unknown) as Twilio; // return mock; // } - const client = new EsClient(await getEsConfig({ config, indexType })); + const client = new EsClient( + await getEsConfig({ config, indexType, ssmConfigParameter }), + ); return { client, index, From d6e187b2875dba43132df41db1bbe7b73e3aa58a Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 21 May 2024 12:35:53 -0300 Subject: [PATCH 30/55] chore: provide ssm parameter name from env vars to ES config --- hrm-domain/lambdas/search-index-consumer/index.ts | 1 - .../search-index-consumer/payloadToIndex.ts | 14 +++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 443fc8c07..4beedcad6 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -14,7 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ import type { SQSBatchResponse, SQSEvent } from 'aws-lambda'; -// import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; import { HrmIndexProcessorError } from '@tech-matters/job-errors'; import { isErr } from '@tech-matters/types'; import { groupMessagesByAccountSid } from './messages'; diff --git a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts index 0e6b96d29..b78928418 100644 --- a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts +++ b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts @@ -110,10 +110,18 @@ const handleIndexPayload = const indexDocumentsByIndexMapper = (accountSid: string) => async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { + if (!process.env.ELASTICSEARCH_CONFIG_PARAMETER) { + throw new Error('ELASTICSEARCH_CONFIG_PARAMETER missing in environment variables'); + } + // get the client for the accountSid-indexType pair - const client = (await getClient({ accountSid, indexType })).indexClient( - hrmIndexConfiguration, - ); + const client = ( + await getClient({ + accountSid, + indexType, + ssmConfigParameter: process.env.ELASTICSEARCH_CONFIG_PARAMETER, + }) + ).indexClient(hrmIndexConfiguration); const mapper = handleIndexPayload({ client, accountSid, indexType }); From 7c51980923cfa77627d6be129c41f60de30f531c Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 21 May 2024 13:17:14 -0300 Subject: [PATCH 31/55] chore: rename env var for ES config --- hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts index b78928418..d3faca0ee 100644 --- a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts +++ b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts @@ -110,8 +110,8 @@ const handleIndexPayload = const indexDocumentsByIndexMapper = (accountSid: string) => async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { - if (!process.env.ELASTICSEARCH_CONFIG_PARAMETER) { - throw new Error('ELASTICSEARCH_CONFIG_PARAMETER missing in environment variables'); + if (!process.env.SSM_PARAM_ELASTICSEARCH_CONFIG) { + throw new Error('SSM_PARAM_ELASTICSEARCH_CONFIG missing in environment variables'); } // get the client for the accountSid-indexType pair @@ -119,7 +119,7 @@ const indexDocumentsByIndexMapper = await getClient({ accountSid, indexType, - ssmConfigParameter: process.env.ELASTICSEARCH_CONFIG_PARAMETER, + ssmConfigParameter: process.env.SSM_PARAM_ELASTICSEARCH_CONFIG, }) ).indexClient(hrmIndexConfiguration); From 217892888d22b9e57f9f4660d03817867521dc56 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 22 May 2024 14:57:32 -0300 Subject: [PATCH 32/55] chore: added hrm-search-config package to hrm-core --- hrm-domain/hrm-core/package.json | 1 + package-lock.json | 2 ++ 2 files changed, 3 insertions(+) diff --git a/hrm-domain/hrm-core/package.json b/hrm-domain/hrm-core/package.json index efe0e6757..055b26c0a 100644 --- a/hrm-domain/hrm-core/package.json +++ b/hrm-domain/hrm-core/package.json @@ -23,6 +23,7 @@ }, "homepage": "https://github.com/tech-matters/hrm#readme", "dependencies": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", diff --git a/package-lock.json b/package-lock.json index 4d68813c4..ef036281f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "version": "1.0.0", "license": "AGPL", "dependencies": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", @@ -17474,6 +17475,7 @@ "@tech-matters/hrm-core": { "version": "file:hrm-domain/hrm-core", "requires": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", From 0b8307ecb6511155333b89599968610bb9e7e52a Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 22 May 2024 14:59:18 -0300 Subject: [PATCH 33/55] chore: added wrapper to publish index-search messages to corresponding queue --- .../jobs/search/publishToSearchIndex.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts new file mode 100644 index 000000000..c37d3e50f --- /dev/null +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { sendSqsMessage } from '@tech-matters/sqs-client'; +import { getSsmParameter } from '../../config/ssmCache'; +import { IndexMessage } from '@tech-matters/hrm-search-config'; +import { CaseService, Contact } from '@tech-matters/hrm-types'; +import { AccountSID } from '@tech-matters/types'; + +const PENDING_INDEX_QUEUE_SSM_PATH = `/${process.env.NODE_ENV}/${ + process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION +}/sqs/jobs/hrm-search-index/queue-url-consumer`; + +const publishToSearchIndex = async (message: IndexMessage) => { + try { + const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); + + return await sendSqsMessage({ + queueUrl, + message: JSON.stringify(message), + }); + } catch (err) { + console.error( + `Error trying to send message to SQS queue ${PENDING_INDEX_QUEUE_SSM_PATH}`, + err, + ); + } +}; + +export const publishContactToSearchIndex = async ({ + accountSid, + contact, + operation, +}: { + accountSid: AccountSID; + contact: Contact; + operation: IndexMessage['operation']; +}) => publishToSearchIndex({ accountSid, type: 'contact', contact, operation }); + +export const publishCaseToSearchIndex = async ({ + accountSid, + case: caseObj, + operation, +}: { + accountSid: AccountSID; + case: CaseService; + operation: IndexMessage['operation']; +}) => publishToSearchIndex({ accountSid, type: 'case', case: caseObj, operation }); From 0d4f1ec313fdfa3f90242fefcdcd396a3bc8f947 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 23 May 2024 19:46:05 -0300 Subject: [PATCH 34/55] chore: added logic to publish updates on contacts into corresponding index-search queue --- hrm-domain/hrm-core/contact/contactService.ts | 38 +++++++++++++++++++ .../hrm-core/contact/sql/contact-get-sql.ts | 3 ++ 2 files changed, 41 insertions(+) diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index 655631639..a91f85fc8 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -69,6 +69,8 @@ import { } from '../sql'; import { systemUser } from '@tech-matters/twilio-worker-auth'; import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; +import type { IndexMessage } from '@tech-matters/hrm-search-config'; +import { publishContactToSearchIndex } from '../jobs/search/publishToSearchIndex'; // Re export as is: export { Contact } from './contactDataAccess'; @@ -172,6 +174,23 @@ const initProfile = async ( }); }; +const doOPContactInSearchIndex = + (operation: IndexMessage['operation']) => + async ({ + accountSid, + contactId, + }: { + accountSid: Contact['accountSid']; + contactId: Contact['id']; + }) => { + const contact = await getById(accountSid, contactId); + + await publishContactToSearchIndex({ accountSid, contact, operation }); + }; + +export const indexContactInSearchIndex = doOPContactInSearchIndex('index'); +const removeContactInSearchIndex = doOPContactInSearchIndex('remove'); + // Creates a contact with all its related records within a single transaction export const createContact = async ( accountSid: HrmAccountId, @@ -220,6 +239,8 @@ export const createContact = async ( return newOk({ data: applyTransformations(contact) }); }); if (isOk(result)) { + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: result.data.id }); return result.data; } // This operation can fail with a unique constraint violation if a contact with the same ID is being created concurrently @@ -243,6 +264,7 @@ export const createContact = async ( return result.unwrap(); } } + return result.unwrap(); }; @@ -287,6 +309,9 @@ export const patchContact = async ( const applyTransformations = bindApplyTransformations(can, user); + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: updated.id }); + return applyTransformations(updated); }); @@ -296,6 +321,11 @@ export const connectContactToCase = async ( caseId: string, { can, user }: { can: InitializedCan; user: TwilioUser }, ): Promise => { + if (caseId === null) { + // trigger remove operation, awaiting for it, since we'll lost the information of which is the "old case" otherwise + await removeContactInSearchIndex({ accountSid, contactId: parseInt(contactId, 10) }); + } + const updated: Contact | undefined = await connectToCase()( accountSid, contactId, @@ -307,6 +337,10 @@ export const connectContactToCase = async ( } const applyTransformations = bindApplyTransformations(can, user); + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: updated.id }); + return applyTransformations(updated); }; @@ -351,6 +385,10 @@ export const addConversationMediaToContact = async ( ...contact, conversationMedia: [...contact.conversationMedia, ...createdConversationMedia], }; + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: updated.id }); + return applyTransformations(updated); }); }; diff --git a/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts b/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts index a421285ba..5cc3468c8 100644 --- a/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts +++ b/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts @@ -21,6 +21,9 @@ import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral- const ID_WHERE_CLAUSE = `WHERE c."accountSid" = $ AND c."id" = $`; const TASKID_WHERE_CLAUSE = `WHERE c."accountSid" = $ AND c."taskId" = $`; +/** + * Note: this query is also used to index Contact records in ES. If the JOINs are ever removed from this query, make sure that the JOINs are preserved for the ES dedicated one + */ export const selectContactsWithRelations = (table: string) => ` SELECT c.*, reports."csamReports", joinedReferrals."referrals", media."conversationMedia" FROM "${table}" c From b165e3fd775e478e40e9e38a8db43c2b8fb1d15c Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 23 May 2024 20:19:40 -0300 Subject: [PATCH 35/55] chore: debug logs --- hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts index c37d3e50f..15ff0ddb2 100644 --- a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -26,6 +26,7 @@ const PENDING_INDEX_QUEUE_SSM_PATH = `/${process.env.NODE_ENV}/${ const publishToSearchIndex = async (message: IndexMessage) => { try { + console.log('>>>> publishToSearchIndex invoked with message: ', message); const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); return await sendSqsMessage({ From 5e4c46875c21ff782d73a00e9d585104f73cab96 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 23 May 2024 20:49:32 -0300 Subject: [PATCH 36/55] chore: copy hrm packages at build time --- hrm-domain/hrm-service/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/hrm-domain/hrm-service/Dockerfile b/hrm-domain/hrm-service/Dockerfile index 1690f12f7..b587c8cc6 100644 --- a/hrm-domain/hrm-service/Dockerfile +++ b/hrm-domain/hrm-service/Dockerfile @@ -35,6 +35,7 @@ RUN apk add --no-cache rsync \ && npx tsc -b tsconfig.build.json \ && cp -r hrm-domain/hrm-service/* /home/node/ \ && mkdir -p /home/node/hrm-domain/ \ + && cp -r hrm-domain/packages /home/node/hrm-domain/packages \ && cp -r hrm-domain/hrm-core /home/node/hrm-domain/hrm-core \ && cp -r hrm-domain/scheduled-tasks /home/node/hrm-domain/scheduled-tasks \ && cp -r packages /home/node/ \ From 029654c8b5534597ea88f59d0fde445ef579233e Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 27 May 2024 15:44:56 -0300 Subject: [PATCH 37/55] chore: add message group id to sqs calls --- .../jobs/search/publishToSearchIndex.ts | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts index 15ff0ddb2..04a8af6c7 100644 --- a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -24,14 +24,24 @@ const PENDING_INDEX_QUEUE_SSM_PATH = `/${process.env.NODE_ENV}/${ process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION }/sqs/jobs/hrm-search-index/queue-url-consumer`; -const publishToSearchIndex = async (message: IndexMessage) => { +const publishToSearchIndex = async ({ + message, + messageGroupId, +}: { + message: IndexMessage; + messageGroupId: string; +}) => { try { - console.log('>>>> publishToSearchIndex invoked with message: ', message); + console.log( + '>>>> publishToSearchIndex invoked with message: ', + JSON.stringify(message), + ); const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); return await sendSqsMessage({ queueUrl, message: JSON.stringify(message), + messageGroupId, }); } catch (err) { console.error( @@ -49,7 +59,11 @@ export const publishContactToSearchIndex = async ({ accountSid: AccountSID; contact: Contact; operation: IndexMessage['operation']; -}) => publishToSearchIndex({ accountSid, type: 'contact', contact, operation }); +}) => + publishToSearchIndex({ + message: { accountSid, type: 'contact', contact, operation }, + messageGroupId: `${accountSid}-contact-${contact.id}`, + }); export const publishCaseToSearchIndex = async ({ accountSid, @@ -59,4 +73,8 @@ export const publishCaseToSearchIndex = async ({ accountSid: AccountSID; case: CaseService; operation: IndexMessage['operation']; -}) => publishToSearchIndex({ accountSid, type: 'case', case: caseObj, operation }); +}) => + publishToSearchIndex({ + message: { accountSid, type: 'case', case: caseObj, operation }, + messageGroupId: `${accountSid}-case-${caseObj.id}`, + }); From 7262aaf46090a880cfa82af75269376922a60780 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 27 May 2024 15:45:20 -0300 Subject: [PATCH 38/55] chore: convertToIndexDocument only filters out undefined values --- .../hrm-search-config/convertToIndexDocument.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index f1ae36ced..238aff6b6 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,13 +23,13 @@ import { import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; import { IndexPayload, IndexPayloadCase, IndexPayloadContact } from './payload'; -const filterEmpty = (doc: T): T => +const filterUndefined = (doc: T): T => Object.entries(doc).reduce((accum, [key, value]) => { - if (value) { - return { ...accum, [key]: value }; + if (value === undefined) { + return accum; } - return accum; + return { ...accum, [key]: value }; }, {} as T); export const convertContactToContactDocument = ({ @@ -71,7 +71,7 @@ export const convertContactToContactDocument = ({ // low_boost_global: '', // lowBoostGlobal.join(' '), }; - return filterEmpty(contactDocument); + return filterUndefined(contactDocument); }; const convertCaseToCaseDocument = ({ @@ -130,7 +130,7 @@ const convertCaseToCaseDocument = ({ // low_boost_global: '', // lowBoostGlobal.join(' '), }; - return filterEmpty(caseDocument); + return filterUndefined(caseDocument); }; const convertToContactIndexDocument = (payload: IndexPayload) => { From 06bfd60b5c602cdf40573d88f2a63a17be6f2fe0 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 27 May 2024 17:01:15 -0300 Subject: [PATCH 39/55] chore: trigger re-index on conversation media updates --- .../contact-job/contact-job-complete.ts | 4 ++-- hrm-domain/hrm-core/contact/contactService.ts | 23 +++++++++++++++++++ .../conversation-media-data-access.ts | 3 +++ .../conversation-media/conversation-media.ts | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/hrm-domain/hrm-core/contact-job/contact-job-complete.ts b/hrm-domain/hrm-core/contact-job/contact-job-complete.ts index 85a32fc49..6b9490840 100644 --- a/hrm-domain/hrm-core/contact-job/contact-job-complete.ts +++ b/hrm-domain/hrm-core/contact-job/contact-job-complete.ts @@ -20,6 +20,7 @@ import { completeContactJob, getContactJobById, } from './contact-job-data-access'; +import { updateConversationMediaData } from '../contact/contactService'; import { ContactJobAttemptResult, ContactJobType } from '@tech-matters/types'; import { ContactJobCompleteProcessorError, @@ -41,7 +42,6 @@ import type { import { ConversationMedia, getConversationMediaById, - updateConversationMediaData, } from '../conversation-media/conversation-media'; export const processCompletedRetrieveContactTranscript = async ( @@ -59,7 +59,7 @@ export const processCompletedRetrieveContactTranscript = async ( location: completedJob.attemptPayload, }; - return updateConversationMediaData( + return updateConversationMediaData(completedJob.contactId)( completedJob.accountSid, completedJob.conversationMediaId, storeTypeSpecificData, diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index a91f85fc8..41b155571 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -59,6 +59,7 @@ import { isS3StoredTranscript, isS3StoredTranscriptPending, NewConversationMedia, + updateConversationMediaSpecificData, } from '../conversation-media/conversation-media'; import { Profile, getOrCreateProfileWithIdentifier } from '../profile/profileService'; import { deleteContactReferrals } from '../referral/referral-data-access'; @@ -463,3 +464,25 @@ export const getContactsByProfileId = async ( }); } }; + +/** + * wrapper around updateSpecificData that also triggers a re-index operation when the conversation media gets updated (e.g. when transcript is exported) + */ +export const updateConversationMediaData = + (contactId: Contact['id']) => + async ( + ...[accountSid, id, storeTypeSpecificData]: Parameters< + typeof updateConversationMediaSpecificData + > + ): ReturnType => { + const result = await updateConversationMediaSpecificData( + accountSid, + id, + storeTypeSpecificData, + ); + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId }); + + return result; + }; diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts index c02a7eeee..0fc7b2d7b 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts @@ -116,6 +116,9 @@ export const getByContactId = async ( }), ); +/** + * NOTE: This function should not be used, but via the wrapper exposed from contact service. This is because otherwise, no contact re-index will be triggered. + */ export const updateSpecificData = async ( accountSid: HrmAccountId, id: ConversationMedia['id'], diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media.ts b/hrm-domain/hrm-core/conversation-media/conversation-media.ts index 73c683519..a403b520e 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media.ts @@ -26,5 +26,5 @@ export { create as createConversationMedia, getById as getConversationMediaById, getByContactId as getConversationMediaByContactId, - updateSpecificData as updateConversationMediaData, + updateSpecificData as updateConversationMediaSpecificData, } from './conversation-media-data-access'; From f8b494fa635d1d5ce79555a9acb0fa15ccb39ad1 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 29 May 2024 12:21:03 -0300 Subject: [PATCH 40/55] chore: fix unit tests --- .../hrm-core/unit-tests/contact/contactService.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts index f0d83bd95..10762ff59 100644 --- a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts +++ b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts @@ -34,6 +34,11 @@ import { ALWAYS_CAN, OPEN_CONTACT_ACTION_CONDITIONS } from '../mocks'; import '@tech-matters/testing/expectToParseAsDate'; import { openPermissions } from '../../permissions/json-permissions'; import { RulesFile, TKConditionsSets } from '../../permissions/rulesMap'; +import * as publishToSearchIndex from '../../jobs/search/publishToSearchIndex'; + +const publishToSearchIndexSpy = jest + .spyOn(publishToSearchIndex, 'publishContactToSearchIndex') + .mockImplementation(async () => Promise.resolve('Ok') as any); const accountSid = 'AC-accountSid'; const workerSid = 'WK-WORKER_SID'; From de33345f91a936648941bf04d1a34168b932d721 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 29 May 2024 12:26:04 -0300 Subject: [PATCH 41/55] chore: added unit tests --- .../unit-tests/contact/contactService.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts index 10762ff59..7e94f06cc 100644 --- a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts +++ b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts @@ -145,6 +145,7 @@ describe('createContact', () => { identifierId: 1, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -173,6 +174,7 @@ describe('createContact', () => { identifierId: 2, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -207,6 +209,7 @@ describe('createContact', () => { identifierId: undefined, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -228,6 +231,7 @@ describe('createContact', () => { identifierId: 1, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -250,6 +254,7 @@ describe('createContact', () => { identifierId: 1, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); }); @@ -266,6 +271,8 @@ describe('connectContactToCase', () => { '4321', ALWAYS_CAN.user.workerSid, ); + + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(result).toStrictEqual(mockContact); }); @@ -276,6 +283,7 @@ describe('connectContactToCase', () => { expect( connectContactToCase(accountSid, '1234', '4321', ALWAYS_CAN), ).rejects.toThrow(); + expect(publishToSearchIndexSpy).not.toHaveBeenCalled(); }); }); @@ -310,6 +318,8 @@ describe('patchContact', () => { samplePatch, ALWAYS_CAN, ); + + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(result).toStrictEqual(mockContact); expect(patchSpy).toHaveBeenCalledWith(accountSid, '1234', true, { updatedBy: contactPatcherSid, @@ -333,6 +343,8 @@ describe('patchContact', () => { const patchSpy = jest.fn(); jest.spyOn(contactDb, 'patch').mockReturnValue(patchSpy); patchSpy.mockResolvedValue(undefined); + + expect(publishToSearchIndexSpy).not.toHaveBeenCalled(); expect( patchContact(accountSid, contactPatcherSid, true, '1234', samplePatch, ALWAYS_CAN), ).rejects.toThrow(); From e655b603bd4c68b4cd77da86afeafaaeda4cada7 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 29 May 2024 15:53:59 -0300 Subject: [PATCH 42/55] chore: fix service tests --- .../service-tests/contact-job/contactJobCleanup.test.ts | 8 +++----- .../contact-job/jobTypes/retrieveTranscript.test.ts | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts b/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts index 07d4346ed..cca2b5846 100644 --- a/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts @@ -24,12 +24,10 @@ import { mockSuccessfulTwilioAuthentication, } from '@tech-matters/testing'; import { createContactJob } from '@tech-matters/hrm-core/contact-job/contact-job-data-access'; -import { - isS3StoredTranscriptPending, - updateConversationMediaData, -} from '@tech-matters/hrm-core/conversation-media/conversation-media'; +import { isS3StoredTranscriptPending } from '@tech-matters/hrm-core/conversation-media/conversation-media'; import { S3ContactMediaType } from '@tech-matters/hrm-core/conversation-media/conversation-media'; import { getById as getContactById } from '@tech-matters/hrm-core/contact/contactDataAccess'; +import { updateConversationMediaData } from '@tech-matters/hrm-core/contact/contactService'; import * as cleanupContactJobsApi from '@tech-matters/contact-job-cleanup'; import { completeContactJob, @@ -206,7 +204,7 @@ describe('cleanupContactJobs', () => { job = await completeContactJob({ id: job.id, completionPayload }); job = await backDateJob(job.id); - await updateConversationMediaData( + await updateConversationMediaData(contact.id)( accountSid, job.additionalPayload.conversationMediaId, completionPayload, diff --git a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts index 69d0af6b8..021944669 100644 --- a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts @@ -515,7 +515,7 @@ describe('complete retrieve-transcript job type', () => { // ); const updateConversationMediaSpy = jest.spyOn( - conversationMediaApi, + contactApi, 'updateConversationMediaData', ); From 8ca4f422ced5b988b3f25c8bc432fc62c79b3a7b Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 30 May 2024 19:43:19 -0300 Subject: [PATCH 43/55] chore: added logic to publish updates on cases into index-search queue --- hrm-domain/hrm-core/case/caseService.ts | 45 +++++++++++++++++++ hrm-domain/hrm-core/contact/contactService.ts | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index 7dce4a355..c2f860cbd 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -48,6 +48,8 @@ import { import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; +import type { IndexMessage } from '@tech-matters/hrm-search-config'; +import { publishCaseToSearchIndex } from '../jobs/search/publishToSearchIndex'; export { WELL_KNOWN_CASE_SECTION_NAMES, CaseService, CaseInfoSection }; @@ -267,6 +269,40 @@ const mapEssentialData = }; }; +// TODO: use the factored out version once that's merged +const maxPermissions: { + user: TwilioUser; + can: () => boolean; +} = { + can: () => true, + user: { + accountSid: 'ACxxx', + workerSid: 'WKxxx', + roles: ['supervisor'], + isSupervisor: true, + }, +}; + +const doOPCaseInSearchIndex = + (operation: IndexMessage['operation']) => + async ({ + accountSid, + caseId, + }: { + accountSid: CaseService['accountSid']; + caseId: CaseService['id']; + }) => { + const caseObj = await getById(caseId, accountSid, maxPermissions.user, []); + + await publishCaseToSearchIndex({ + accountSid, + case: caseRecordToCase(caseObj), + operation, + }); + }; + +export const indexCaseInSearchIndex = doOPCaseInSearchIndex('index'); + export const createCase = async ( body: Partial, accountSid: CaseService['accountSid'], @@ -289,6 +325,9 @@ export const createCase = async ( ); const created = await create(record); + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: created.id }); + // A new case is always initialized with empty connected contacts. No need to apply mapContactTransformations here return caseRecordToCase(created); }; @@ -315,6 +354,9 @@ export const updateCaseStatus = async ( const withTransformedContacts = mapContactTransformations({ can, user })(updated); + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: updated.id }); + return caseRecordToCase(withTransformedContacts); }; @@ -327,6 +369,9 @@ export const updateCaseOverview = async ( const validOverview = pick(overview, CASE_OVERVIEW_PROPERTIES); const updated = await updateCaseInfo(accountSid, id, validOverview, workerSid); + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: updated.id }); + return caseRecordToCase(updated); }; diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index 41b155571..95994ef65 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -189,7 +189,7 @@ const doOPContactInSearchIndex = await publishContactToSearchIndex({ accountSid, contact, operation }); }; -export const indexContactInSearchIndex = doOPContactInSearchIndex('index'); +const indexContactInSearchIndex = doOPContactInSearchIndex('index'); const removeContactInSearchIndex = doOPContactInSearchIndex('remove'); // Creates a contact with all its related records within a single transaction From 50d068efaabebf3f108df76c4cc99eb50a0c0dea Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 30 May 2024 19:43:29 -0300 Subject: [PATCH 44/55] chore: added logic to publish updates on case sections into index-search queue --- .../case/caseSection/caseSectionService.ts | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts b/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts index 520ccabc8..e6b9ae0ea 100644 --- a/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts +++ b/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts @@ -36,6 +36,7 @@ import { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { RulesFile, TKConditionsSets } from '../../permissions/rulesMap'; import { ListConfiguration } from '../caseDataAccess'; import { HrmAccountId } from '@tech-matters/types'; +import { indexCaseInSearchIndex } from '../caseService'; const sectionRecordToSection = ( sectionRecord: CaseSectionRecord | undefined, @@ -65,7 +66,13 @@ export const createCaseSection = async ( createdAt: nowISO, accountSid, }; - return sectionRecordToSection(await create()(record)); + + const created = await create()(record); + + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: created.caseId }); + + return sectionRecordToSection(created); }; export const replaceCaseSection = async ( @@ -83,15 +90,19 @@ export const replaceCaseSection = async ( updatedBy: workerSid, updatedAt: nowISO, }; - return sectionRecordToSection( - await updateById()( - accountSid, - Number.parseInt(caseId), - sectionType, - sectionId, - record, - ), + + const updated = await updateById()( + accountSid, + Number.parseInt(caseId), + sectionType, + sectionId, + record, ); + + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: updated.caseId }); + + return sectionRecordToSection(updated); }; export const getCaseSection = async ( @@ -155,13 +166,16 @@ export const deleteCaseSection = async ( sectionId: string, { user }: { user: TwilioUser }, ): Promise => { - return sectionRecordToSection( - await deleteById()( - accountSid, - Number.parseInt(caseId), - sectionType, - sectionId, - user.workerSid, - ), + const deleted = await deleteById()( + accountSid, + Number.parseInt(caseId), + sectionType, + sectionId, + user.workerSid, ); + + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: deleted.caseId }); + + return sectionRecordToSection(deleted); }; From beb86807f7c464bb70dfbf1221550ed3c6b49eab Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 31 May 2024 13:52:01 -0300 Subject: [PATCH 45/55] chore: fix unit tests --- .../hrm-core/unit-tests/case/caseService.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts b/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts index 519766a9b..74fd669d0 100644 --- a/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts +++ b/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts @@ -24,6 +24,11 @@ import { workerSid, accountSid } from '../mocks'; import { newTwilioUser } from '@tech-matters/twilio-worker-auth'; import { rulesMap } from '../../permissions'; import { RulesFile } from '../../permissions/rulesMap'; +import * as publishToSearchIndex from '../../jobs/search/publishToSearchIndex'; + +const publishToSearchIndexSpy = jest + .spyOn(publishToSearchIndex, 'publishCaseToSearchIndex') + .mockImplementation(async () => Promise.resolve('Ok') as any); jest.mock('../../case/caseDataAccess'); const baselineCreatedDate = new Date(2013, 6, 13).toISOString(); @@ -54,6 +59,8 @@ test('create case', async () => { accountSid, }; const createSpy = jest.spyOn(caseDb, 'create').mockResolvedValue(createdCaseRecord); + // const getByIdSpy = + jest.spyOn(caseDb, 'getById').mockResolvedValueOnce(createdCaseRecord); const createdCase = await caseApi.createCase(caseToBeCreated, accountSid, workerSid); // any worker & account specified on the object should be overwritten with the ones from the user @@ -68,6 +75,11 @@ test('create case', async () => { userOwnsContact: false, }, }); + + await Promise.resolve('Iddle'); + await Promise.resolve('Iddle'); + + expect(publishToSearchIndexSpy).toHaveBeenCalled(); }); describe('searchCases', () => { From 28a47dc9380527a1ebf3d8c9f79b2d4cccec8b82 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 31 May 2024 14:40:48 -0300 Subject: [PATCH 46/55] chore: naming nit --- hrm-domain/hrm-core/case/caseService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index c2f860cbd..bead7c619 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -283,7 +283,7 @@ const maxPermissions: { }, }; -const doOPCaseInSearchIndex = +const doCaseInSearchIndexOP = (operation: IndexMessage['operation']) => async ({ accountSid, @@ -301,7 +301,7 @@ const doOPCaseInSearchIndex = }); }; -export const indexCaseInSearchIndex = doOPCaseInSearchIndex('index'); +export const indexCaseInSearchIndex = doCaseInSearchIndexOP('index'); export const createCase = async ( body: Partial, From c7ba6c5976c2140f950611e78fefa0520b02c12e Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 31 May 2024 14:41:26 -0300 Subject: [PATCH 47/55] chore: PR feedback (naming nit) --- hrm-domain/hrm-core/contact/contactService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index 41b155571..ca578afb4 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -175,7 +175,7 @@ const initProfile = async ( }); }; -const doOPContactInSearchIndex = +const doContactInSearchIndexOP = (operation: IndexMessage['operation']) => async ({ accountSid, @@ -189,8 +189,8 @@ const doOPContactInSearchIndex = await publishContactToSearchIndex({ accountSid, contact, operation }); }; -export const indexContactInSearchIndex = doOPContactInSearchIndex('index'); -const removeContactInSearchIndex = doOPContactInSearchIndex('remove'); +export const indexContactInSearchIndex = doContactInSearchIndexOP('index'); +const removeContactInSearchIndex = doContactInSearchIndexOP('remove'); // Creates a contact with all its related records within a single transaction export const createContact = async ( From b7ef6ee0fd749c184c5f98a2a605f55989dc5eeb Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 31 May 2024 16:57:13 -0300 Subject: [PATCH 48/55] chore: add support for deleteDocument OP in elasticsearch client package --- packages/elasticsearch-client/src/client.ts | 7 ++++ .../src/deleteDocument.ts | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 packages/elasticsearch-client/src/deleteDocument.ts diff --git a/packages/elasticsearch-client/src/client.ts b/packages/elasticsearch-client/src/client.ts index 265d76502..385b7502f 100644 --- a/packages/elasticsearch-client/src/client.ts +++ b/packages/elasticsearch-client/src/client.ts @@ -35,6 +35,10 @@ import { search, SearchExtraParams } from './search'; import { suggest, SuggestExtraParams } from './suggest'; import { SearchConfiguration, IndexConfiguration } from './config'; import { IndicesRefreshResponse } from '@elastic/elasticsearch/lib/api/types'; +import deleteDocument, { + DeleteDocumentExtraParams, + DeleteDocumentResponse, +} from './deleteDocument'; // import { getMockClient } from './mockClient'; type AccountSidOrShortCodeRequired = @@ -108,6 +112,7 @@ export type IndexClient = { indexDocument: (args: IndexDocumentExtraParams) => Promise; updateDocument: (args: UpdateDocumentExtraParams) => Promise; updateScript: (args: UpdateScriptExtraParams) => Promise; + deleteDocument: (args: DeleteDocumentExtraParams) => Promise; refreshIndex: () => Promise; executeBulk: (args: ExecuteBulkExtraParams) => Promise; createIndex: (args: CreateIndexExtraParams) => Promise; @@ -160,6 +165,8 @@ const getClientOrMock = async ({ updateDocument({ ...passThroughConfig, ...args }), updateScript: (args: UpdateScriptExtraParams) => updateScript({ ...passThroughConfig, ...args }), + deleteDocument: (args: DeleteDocumentExtraParams) => + deleteDocument({ ...passThroughConfig, ...args }), executeBulk: (args: ExecuteBulkExtraParams) => executeBulk({ ...passThroughConfig, ...args }), }; diff --git a/packages/elasticsearch-client/src/deleteDocument.ts b/packages/elasticsearch-client/src/deleteDocument.ts new file mode 100644 index 000000000..de54a52e8 --- /dev/null +++ b/packages/elasticsearch-client/src/deleteDocument.ts @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ +import { DeleteResponse } from '@elastic/elasticsearch/lib/api/types'; +import { PassThroughConfig } from './client'; + +export type DeleteDocumentExtraParams = { + id: string; +}; + +export type DeleteDocumentParams = PassThroughConfig & DeleteDocumentExtraParams; +export type DeleteDocumentResponse = DeleteResponse; + +export const deleteDocument = async ({ + client, + id, + index, +}: DeleteDocumentParams): Promise => { + return client.delete({ + index, + id, + }); +}; + +export default deleteDocument; From f553d183c9ef062a95754cca0e1536c6297b45da Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 31 May 2024 16:58:30 -0300 Subject: [PATCH 49/55] chore: add logic to handle remove case op in search-index-consumer --- .../messagesToPayloads.ts | 103 ++++++++++++------ .../search-index-consumer/payloadToIndex.ts | 12 ++ 2 files changed, 80 insertions(+), 35 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts index 0fc19ae9c..8434cd95c 100644 --- a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -36,7 +36,7 @@ export type PayloadWithMeta = { documentId: number; payload: IndexPayload; messageId: string; - indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript'; + indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript' | 'deleteDocument'; }; export type PayloadsByIndex = { [indexType: string]: PayloadWithMeta[]; @@ -111,49 +111,82 @@ const generatePayloadFromContact = ( ps: PayloadsByIndex, m: ContactIndexingInputData, ): PayloadsByIndex => { - return { - ...ps, - // add an upsert job to HRM_CONTACTS_INDEX_TYPE index - [HRM_CONTACTS_INDEX_TYPE]: [ - ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), - { - ...m, - documentId: m.message.contact.id, - payload: { ...m.message, transcript: m.transcript }, - indexHandler: 'updateDocument', - }, - ], - // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId - ? [ - ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + switch (m.message.operation) { + // both operations are handled internally by the hrm-search-config package, so just cascade the cases + case 'index': + case 'remove': { + return { + ...ps, + // add an upsert job to HRM_CONTACTS_INDEX_TYPE index + [HRM_CONTACTS_INDEX_TYPE]: [ + ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), { ...m, - documentId: parseInt(m.message.contact.caseId, 10), + documentId: m.message.contact.id, payload: { ...m.message, transcript: m.transcript }, - indexHandler: 'updateScript', + indexHandler: 'updateDocument', }, - ] - : ps[HRM_CASES_INDEX_TYPE] ?? [], - }; + ], + // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId + ? [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: parseInt(m.message.contact.caseId, 10), + payload: { ...m.message, transcript: m.transcript }, + indexHandler: 'updateScript', + }, + ] + : ps[HRM_CASES_INDEX_TYPE] ?? [], + }; + } + default: { + return assertExhaustive(m.message.operation); + } + } }; const generatePayloadFromCase = ( ps: PayloadsByIndex, m: CaseIndexingInputData, -): PayloadsByIndex => ({ - ...ps, - // add an upsert job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: [ - ...(ps[HRM_CASES_INDEX_TYPE] ?? []), - { - ...m, - documentId: m.message.case.id, - payload: { ...m.message }, - indexHandler: 'updateDocument', - }, - ], -}); +): PayloadsByIndex => { + switch (m.message.operation) { + case 'index': { + return { + ...ps, + // add an upsert job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.case.id, + payload: { ...m.message }, + indexHandler: 'updateDocument', + }, + ], + }; + } + case 'remove': { + return { + ...ps, + // add a delete job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.case.id, + payload: { ...m.message }, + indexHandler: 'deleteDocument', + }, + ], + }; + } + default: { + return assertExhaustive(m.message.operation); + } + } +}; const messagesToPayloadReducer = ( accum: PayloadsByIndex, diff --git a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts index d3faca0ee..6575e5e1b 100644 --- a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts +++ b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts @@ -84,6 +84,18 @@ const handleIndexPayload = result: newOkFromData(result), }; } + case 'deleteDocument': { + const result = await client.deleteDocument({ + id: documentId.toString(), + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } default: { return assertExhaustive(indexHandler); } From d0c1f4af9595977af78374efc65707ab99c6c116 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 31 May 2024 16:59:34 -0300 Subject: [PATCH 50/55] chore: trigger 'remove case from index' op when a case is deleted in HRM --- hrm-domain/hrm-core/case/caseRoutesV0.ts | 3 +-- hrm-domain/hrm-core/case/caseService.ts | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/hrm-domain/hrm-core/case/caseRoutesV0.ts b/hrm-domain/hrm-core/case/caseRoutesV0.ts index b3aca4ace..b7751f610 100644 --- a/hrm-domain/hrm-core/case/caseRoutesV0.ts +++ b/hrm-domain/hrm-core/case/caseRoutesV0.ts @@ -15,7 +15,6 @@ */ import createError from 'http-errors'; -import * as casesDb from './caseDataAccess'; import * as caseApi from './caseService'; import { publicEndpoint, SafeRouter } from '../permissions'; import { @@ -93,7 +92,7 @@ casesRouter.expressRouter.use('/:caseId/sections', caseSectionRoutesV0); casesRouter.delete('/:id', publicEndpoint, async (req, res) => { const { hrmAccountId } = req; const { id } = req.params; - const deleted = await casesDb.deleteById(id, hrmAccountId); + const deleted = await caseApi.deleteCaseById({ accountSid: hrmAccountId, caseId: id }); if (!deleted) { throw createError(404); } diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index bead7c619..edd584fb0 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -32,6 +32,7 @@ import { updateStatus, CaseRecordUpdate, updateCaseInfo, + deleteById, } from './caseDataAccess'; import { randomUUID } from 'crypto'; import { InitializedCan } from '../permissions/initializeCanForRules'; @@ -302,6 +303,7 @@ const doCaseInSearchIndexOP = }; export const indexCaseInSearchIndex = doCaseInSearchIndexOP('index'); +const removeCaseInSearchIndex = doCaseInSearchIndexOP('remove'); export const createCase = async ( body: Partial, @@ -504,3 +506,20 @@ export const getCasesByProfileId = async ( }); } }; + +export const deleteCaseById = async ({ + accountSid, + caseId, +}: { + accountSid: HrmAccountId; + caseId: number; +}) => { + const deleted = await deleteById(caseId, accountSid); + + if (deleted) { + // trigger remove operation but don't await for it + removeCaseInSearchIndex({ accountSid, caseId: deleted.id }); + } + + return deleted; +}; From 51ffc2d4123dea931a535cdf16e210a4f0533e1b Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 3 Jun 2024 18:46:13 -0300 Subject: [PATCH 51/55] chore: handle errors, avoid publishing if there's no record --- hrm-domain/hrm-core/case/caseDataAccess.ts | 7 ++-- .../case/caseSection/caseSectionService.ts | 6 ++-- hrm-domain/hrm-core/case/caseService.ts | 32 ++++++++++++------- hrm-domain/hrm-core/contact/contactService.ts | 19 ++++++++--- 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/hrm-domain/hrm-core/case/caseDataAccess.ts b/hrm-domain/hrm-core/case/caseDataAccess.ts index 376d70aa3..0ae995971 100644 --- a/hrm-domain/hrm-core/case/caseDataAccess.ts +++ b/hrm-domain/hrm-core/case/caseDataAccess.ts @@ -29,10 +29,11 @@ import { Contact } from '../contact/contactDataAccess'; import { DateFilter, OrderByDirectionType } from '../sql'; import { TKConditionsSets } from '../permissions/rulesMap'; import { TwilioUser } from '@tech-matters/twilio-worker-auth'; -import { TwilioUserIdentifier } from '@tech-matters/types'; +import { AccountSID, TwilioUserIdentifier } from '@tech-matters/types'; import { PrecalculatedCasePermissionConditions, CaseRecordCommon, + CaseService, } from '@tech-matters/hrm-types'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; @@ -249,8 +250,8 @@ export const searchByProfileId = generalizedSearchQueryFunction<{ }), ); -export const deleteById = async (id, accountSid) => { - return db.oneOrNone(DELETE_BY_ID, [accountSid, id]); +export const deleteById = async (id: CaseService['id'], accountSid: AccountSID) => { + return db.oneOrNone(DELETE_BY_ID, [accountSid, id]); }; export const updateStatus = async ( diff --git a/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts b/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts index e6b9ae0ea..25c00d4b3 100644 --- a/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts +++ b/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts @@ -70,7 +70,7 @@ export const createCaseSection = async ( const created = await create()(record); // trigger index operation but don't await for it - indexCaseInSearchIndex({ accountSid, caseId: created.caseId }); + indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) }); return sectionRecordToSection(created); }; @@ -100,7 +100,7 @@ export const replaceCaseSection = async ( ); // trigger index operation but don't await for it - indexCaseInSearchIndex({ accountSid, caseId: updated.caseId }); + indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) }); return sectionRecordToSection(updated); }; @@ -175,7 +175,7 @@ export const deleteCaseSection = async ( ); // trigger index operation but don't await for it - indexCaseInSearchIndex({ accountSid, caseId: deleted.caseId }); + indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) }); return sectionRecordToSection(deleted); }; diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index edd584fb0..4df906cc6 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -289,17 +289,29 @@ const doCaseInSearchIndexOP = async ({ accountSid, caseId, + caseRecord, }: { accountSid: CaseService['accountSid']; caseId: CaseService['id']; + caseRecord?: CaseRecord; }) => { - const caseObj = await getById(caseId, accountSid, maxPermissions.user, []); - - await publishCaseToSearchIndex({ - accountSid, - case: caseRecordToCase(caseObj), - operation, - }); + try { + const caseObj = + caseRecord || (await getById(caseId, accountSid, maxPermissions.user, [])); + + if (caseObj) { + await publishCaseToSearchIndex({ + accountSid, + case: caseRecordToCase(caseObj), + operation, + }); + } + } catch (err) { + console.error( + `Error trying to index case: accountSid ${accountSid} caseId ${caseId}`, + err, + ); + } }; export const indexCaseInSearchIndex = doCaseInSearchIndexOP('index'); @@ -516,10 +528,8 @@ export const deleteCaseById = async ({ }) => { const deleted = await deleteById(caseId, accountSid); - if (deleted) { - // trigger remove operation but don't await for it - removeCaseInSearchIndex({ accountSid, caseId: deleted.id }); - } + // trigger remove operation but don't await for it + removeCaseInSearchIndex({ accountSid, caseId: deleted.id, caseRecord: deleted }); return deleted; }; diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index 95994ef65..bd3cb1837 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -184,9 +184,18 @@ const doOPContactInSearchIndex = accountSid: Contact['accountSid']; contactId: Contact['id']; }) => { - const contact = await getById(accountSid, contactId); + try { + const contact = await getById(accountSid, contactId); - await publishContactToSearchIndex({ accountSid, contact, operation }); + if (contact) { + await publishContactToSearchIndex({ accountSid, contact, operation }); + } + } catch (err) { + console.error( + `Error trying to index contact: accountSid ${accountSid} contactId ${contactId}`, + err, + ); + } }; const indexContactInSearchIndex = doOPContactInSearchIndex('index'); @@ -311,7 +320,7 @@ export const patchContact = async ( const applyTransformations = bindApplyTransformations(can, user); // trigger index operation but don't await for it - indexContactInSearchIndex({ accountSid, contactId: updated.id }); + indexContactInSearchIndex({ accountSid, contactId: parseInt(contactId, 10) }); return applyTransformations(updated); }); @@ -340,7 +349,7 @@ export const connectContactToCase = async ( const applyTransformations = bindApplyTransformations(can, user); // trigger index operation but don't await for it - indexContactInSearchIndex({ accountSid, contactId: updated.id }); + indexContactInSearchIndex({ accountSid, contactId: parseInt(contactId, 10) }); return applyTransformations(updated); }; @@ -388,7 +397,7 @@ export const addConversationMediaToContact = async ( }; // trigger index operation but don't await for it - indexContactInSearchIndex({ accountSid, contactId: updated.id }); + indexContactInSearchIndex({ accountSid, contactId: parseInt(contactIdString, 10) }); return applyTransformations(updated); }); From c228b35519f09e4f9b9b925b544802b15ad9f6da Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 4 Jun 2024 17:33:28 -0300 Subject: [PATCH 52/55] chore: fix unit tests --- .../unit-tests/case/caseService.test.ts | 4 +- .../unit-tests/contact/contactService.test.ts | 48 +++++++++++++------ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts b/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts index 74fd669d0..b05d91851 100644 --- a/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts +++ b/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts @@ -76,9 +76,7 @@ test('create case', async () => { }, }); - await Promise.resolve('Iddle'); - await Promise.resolve('Iddle'); - + await new Promise(process.nextTick); expect(publishToSearchIndexSpy).toHaveBeenCalled(); }); diff --git a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts index 7e94f06cc..588c51f25 100644 --- a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts +++ b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts @@ -36,6 +36,12 @@ import { openPermissions } from '../../permissions/json-permissions'; import { RulesFile, TKConditionsSets } from '../../permissions/rulesMap'; import * as publishToSearchIndex from '../../jobs/search/publishToSearchIndex'; +const flushPromises = async () => { + await new Promise(process.nextTick); + await new Promise(process.nextTick); + await new Promise(process.nextTick); +}; + const publishToSearchIndexSpy = jest .spyOn(publishToSearchIndex, 'publishContactToSearchIndex') .mockImplementation(async () => Promise.resolve('Ok') as any); @@ -117,21 +123,26 @@ describe('createContact', () => { }; const spyOnContact = ({ - contactMockReturn, + mocks, }: { - contactMockReturn?: ReturnType; + mocks?: { + contactMockReturn: ReturnType; + getContactMock: contactDb.Contact; + }; } = {}) => { - const createContactMock = jest.fn( - contactMockReturn || - (() => Promise.resolve({ contact: mockContact, isNewRecord: true })), - ); - jest.spyOn(contactDb, 'create').mockReturnValue(createContactMock); - - return createContactMock; + const createContactMock = mocks + ? jest.fn(mocks.contactMockReturn) + : jest.fn(() => Promise.resolve({ contact: mockContact, isNewRecord: true })); + const createSpy = jest.spyOn(contactDb, 'create').mockReturnValue(createContactMock); + const getByIdSpy = mocks + ? jest.spyOn(contactDb, 'getById').mockResolvedValueOnce(mocks.getContactMock) + : jest.spyOn(contactDb, 'getById').mockResolvedValueOnce(mockContact); + + return { createContactMock, createSpy, getByIdSpy }; }; test("Passes payload down to data layer with user workerSid used for 'createdBy'", async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); const returnValue = await createContact( parameterAccountSid, 'WK-contact-creator', @@ -145,12 +156,13 @@ describe('createContact', () => { identifierId: 1, }); + await flushPromises(); expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); test("If no identifier record exists for 'number', call createIdentifierAndProfile", async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); getIdentifierWithProfilesSpy.mockImplementationOnce(() => async () => null); @@ -174,12 +186,13 @@ describe('createContact', () => { identifierId: 2, }); + await flushPromises(); expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); test('Missing values are converted to empty strings for several fields', async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); const minimalPayload = omit( sampleCreateContactPayload, @@ -209,12 +222,13 @@ describe('createContact', () => { identifierId: undefined, }); + await flushPromises(); expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); test('Missing timeOfContact value is substituted with current date', async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); const payload = omit(sampleCreateContactPayload, 'timeOfContact'); const returnValue = await createContact( @@ -231,12 +245,13 @@ describe('createContact', () => { identifierId: 1, }); + await flushPromises(); expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); test('queue will be empty if not present', async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); const payload = omit(sampleCreateContactPayload, 'queueName'); const legacyPayload = omit(sampleCreateContactPayload, 'queueName'); @@ -254,6 +269,7 @@ describe('createContact', () => { identifierId: 1, }); + await flushPromises(); expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -262,6 +278,7 @@ describe('createContact', () => { describe('connectContactToCase', () => { test('Returns contact produced by data access layer', async () => { const connectSpy = jest.fn(); + jest.spyOn(contactDb, 'getById').mockResolvedValueOnce(mockContact); connectSpy.mockResolvedValue(mockContact); jest.spyOn(contactDb, 'connectToCase').mockImplementation(() => connectSpy); const result = await connectContactToCase(accountSid, '1234', '4321', ALWAYS_CAN); @@ -272,6 +289,7 @@ describe('connectContactToCase', () => { ALWAYS_CAN.user.workerSid, ); + await flushPromises(); expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(result).toStrictEqual(mockContact); }); @@ -309,6 +327,7 @@ describe('patchContact', () => { test('Passes callerInformation, childInformation, caseInformation & categories to data layer as separate properties', async () => { const patchSpy = jest.fn(); jest.spyOn(contactDb, 'patch').mockReturnValue(patchSpy); + jest.spyOn(contactDb, 'getById').mockResolvedValueOnce(mockContact); patchSpy.mockResolvedValue(mockContact); const result = await patchContact( accountSid, @@ -319,6 +338,7 @@ describe('patchContact', () => { ALWAYS_CAN, ); + await flushPromises(); expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(result).toStrictEqual(mockContact); expect(patchSpy).toHaveBeenCalledWith(accountSid, '1234', true, { From 9f33296a872db20433dfd524832e21126e044fb5 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 4 Jun 2024 17:50:41 -0300 Subject: [PATCH 53/55] chore: contemplate possible undefined deleted case --- hrm-domain/hrm-core/case/caseService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index 4df906cc6..03bd0f677 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -529,7 +529,7 @@ export const deleteCaseById = async ({ const deleted = await deleteById(caseId, accountSid); // trigger remove operation but don't await for it - removeCaseInSearchIndex({ accountSid, caseId: deleted.id, caseRecord: deleted }); + removeCaseInSearchIndex({ accountSid, caseId: deleted?.id, caseRecord: deleted }); return deleted; }; From ea0dcd4d6e0229cd845644efb6ea6ab9c33af138 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 4 Jun 2024 20:37:48 -0300 Subject: [PATCH 54/55] chore: add FF to avoid introducing breaking code --- hrm-domain/hrm-core/case/caseService.ts | 5 +++++ hrm-domain/hrm-core/contact/contactService.ts | 9 ++++++++- hrm-domain/hrm-core/featureFlags.ts | 3 +++ hrm-domain/hrm-core/setTestEnvVars.js | 1 + hrm-domain/hrm-service/setTestEnvVars.js | 1 + 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index 03bd0f677..3353f759b 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -51,6 +51,7 @@ import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; import type { IndexMessage } from '@tech-matters/hrm-search-config'; import { publishCaseToSearchIndex } from '../jobs/search/publishToSearchIndex'; +import { enablePublishHrmSearchIndex } from '../featureFlags'; export { WELL_KNOWN_CASE_SECTION_NAMES, CaseService, CaseInfoSection }; @@ -296,6 +297,10 @@ const doCaseInSearchIndexOP = caseRecord?: CaseRecord; }) => { try { + if (!enablePublishHrmSearchIndex) { + return; + } + const caseObj = caseRecord || (await getById(caseId, accountSid, maxPermissions.user, [])); diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index bd3cb1837..2296f5a8f 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -51,7 +51,10 @@ import type { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { createReferral } from '../referral/referral-model'; import { createContactJob } from '../contact-job/contact-job'; import { isChatChannel } from '@tech-matters/hrm-types'; -import { enableCreateContactJobsFlag } from '../featureFlags'; +import { + enableCreateContactJobsFlag, + enablePublishHrmSearchIndex, +} from '../featureFlags'; import { db } from '../connection-pool'; import { ConversationMedia, @@ -185,6 +188,10 @@ const doOPContactInSearchIndex = contactId: Contact['id']; }) => { try { + if (!enablePublishHrmSearchIndex) { + return; + } + const contact = await getById(accountSid, contactId); if (contact) { diff --git a/hrm-domain/hrm-core/featureFlags.ts b/hrm-domain/hrm-core/featureFlags.ts index 7a1dd0ed3..61116fdf1 100644 --- a/hrm-domain/hrm-core/featureFlags.ts +++ b/hrm-domain/hrm-core/featureFlags.ts @@ -24,3 +24,6 @@ export const enableCleanupJobs = /^true$/i.test(process.env.ENABLE_CLEANUP_JOBS) export const enableProfileFlagsCleanup = /^true$/i.test( process.env.ENABLE_PROFILE_FLAGS_CLEANUP, ); +export const enablePublishHrmSearchIndex = /^true$/i.test( + process.env.ENABLE_PUBLISH_HRM_SEARCH_INDEX, +); diff --git a/hrm-domain/hrm-core/setTestEnvVars.js b/hrm-domain/hrm-core/setTestEnvVars.js index 451d6a681..a2d53b13e 100644 --- a/hrm-domain/hrm-core/setTestEnvVars.js +++ b/hrm-domain/hrm-core/setTestEnvVars.js @@ -45,6 +45,7 @@ process.env.STATIC_KEY_ACCOUNT_SID = 'BBC'; process.env.INCLUDE_ERROR_IN_RESPONSE = true; +process.env.ENABLE_PUBLISH_HRM_SEARCH_INDEX = true; process.env.ENABLE_CREATE_CONTACT_JOBS = true; process.env.ENABLE_PROCESS_CONTACT_JOBS = true; process.env.ENABLE_CLEANUP_JOBS = true; diff --git a/hrm-domain/hrm-service/setTestEnvVars.js b/hrm-domain/hrm-service/setTestEnvVars.js index 55e7ce4d5..fa0256685 100644 --- a/hrm-domain/hrm-service/setTestEnvVars.js +++ b/hrm-domain/hrm-service/setTestEnvVars.js @@ -43,6 +43,7 @@ process.env.STATIC_KEY_ACCOUNT_SID = 'BBC'; process.env.INCLUDE_ERROR_IN_RESPONSE = true; +process.env.ENABLE_PUBLISH_HRM_SEARCH_INDEX = true; process.env.ENABLE_CREATE_CONTACT_JOBS = true; process.env.ENABLE_PROCESS_CONTACT_JOBS = true; process.env.ENABLE_CLEANUP_JOBS = true; From 0b90edce9f8fd5bdc6356b7361eceb8a629b2b73 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 5 Jun 2024 18:04:39 -0300 Subject: [PATCH 55/55] chore: attempt to mock SQS in hrmservice --- cdk/hrm-search-index-stack.ts | 154 ++++++++++++++++++ cdk/init-stack.ts | 13 ++ .../hrm-service/service-tests/case.test.ts | 35 +++- hrm-domain/hrm-service/setTestEnvVars.js | 1 + packages/testing/index.ts | 1 + packages/testing/mockSQS.ts | 77 +++++++++ 6 files changed, 274 insertions(+), 7 deletions(-) create mode 100644 cdk/hrm-search-index-stack.ts create mode 100644 packages/testing/mockSQS.ts diff --git a/cdk/hrm-search-index-stack.ts b/cdk/hrm-search-index-stack.ts new file mode 100644 index 000000000..210182d22 --- /dev/null +++ b/cdk/hrm-search-index-stack.ts @@ -0,0 +1,154 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import * as cdk from 'aws-cdk-lib'; +import * as lambdaNode from 'aws-cdk-lib/aws-lambda-nodejs'; +import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; + +export default class ContactRetrieveStack extends cdk.Stack { + constructor({ + scope, + id, + params = { + skipLambda: false, + }, + props, + }: { + scope: cdk.Construct; + id: string; + params: { + completeQueue: sqs.Queue; + docsBucket: s3.Bucket; + skipLambda?: boolean; + }; + props?: cdk.StackProps; + }) { + super(scope, id, props); + + const queue = new cdk.aws_sqs.Queue(this, id, { + deadLetterQueue: { maxReceiveCount: 1, queue: params.completeQueue }, + }); + + new cdk.CfnOutput(this, `queueUrl`, { + value: queue.queueUrl, + description: `The url of the ${id} queue`, + }); + + new cdk.aws_ssm.StringParameter(this, `queue-url-consumer`, { + parameterName: `/local/us-east-1/sqs/jobs/hrm-search-index/queue-url-consumer`, + stringValue: queue.queueUrl, + }); + + // duplicated for test env + new cdk.aws_ssm.StringParameter(this, `queue-url-consumer-test`, { + parameterName: `/local/us-east-1/sqs/jobs/hrm-search-index/queue-url-consumer-test`, + stringValue: queue.queueUrl, + }); + + if (params.skipLambda) return; + + /* + Here, there be dragons. + + To use the queue urls from inside of a lambda, we have to replace + 'localhost' with 'localstack' so that container to container dns + lookups resolve correctly on the docker network. + + This is WAY more complicated than it should be and took way too long + to figure out. But here goes: + + CDK passes around tokens that are used inside of the final CloudFormation + template that is generated and deployed, not the actual string values for + things like queue urls that aren't known until the deployment is partially + complete. + + Tokens are unresolvable until they are applied in the produced CF template. + So, you can't just do normal string replace operations. You have to use + native cloudformation functions to manipulate the string. + + BUUUT. There is no "replace" function in cloudformation. So you have + to use split/join to do a janky replace. + + Also... for some reason we can't use "Fn::split" inside of a "Fn::Join" + function directly. We have to use the select to iterate over items in the + split or else we just get a string of "Fn::split" as our url. I have no idea + why, but i discovered this working pattern by trial and error mixed with reviewing + generate cloudformation templates from the CDK that used the join/split replace + pattern. + + This is pretty fragile since we can't arbitrarily split/join if there are + multiple instances of the needle in the haystack. But it works for this + simple case. + (rbd 08/10/22) + */ + const splitCompleteQueueUrl = cdk.Fn.split( + 'localhost', + params.completeQueue.queueUrl, + ); + const completedQueueUrl = cdk.Fn.join('localstack', [ + cdk.Fn.select(0, splitCompleteQueueUrl), + cdk.Fn.select(1, splitCompleteQueueUrl), + ]); + + const fn = new lambdaNode.NodejsFunction(this, 'search-index-consumer', { + // TODO: change this back to 18 once it isn't broken upstream + runtime: cdk.aws_lambda.Runtime.NODEJS_16_X, + memorySize: 512, + timeout: cdk.Duration.seconds(10), + handler: 'handler', + entry: `./hrm-domain/lambdas/search-index-consumer/index.ts`, + environment: { + NODE_OPTIONS: '--enable-source-maps', + S3_ENDPOINT: 'http://localstack:4566', + S3_FORCE_PATH_STYLE: 'true', + S3_REGION: 'us-east-1', + SSM_ENDPOINT: 'http://localstack:4566', + SQS_ENDPOINT: 'http://localstack:4566', + NODE_ENV: 'local', + completed_sqs_queue_url: completedQueueUrl, + }, + bundling: { sourceMap: true }, + deadLetterQueueEnabled: true, + deadLetterQueue: params.completeQueue, + }); + + fn.addEventSource( + new SqsEventSource(queue, { batchSize: 10, reportBatchItemFailures: true }), + ); + + fn.addToRolePolicy( + new cdk.aws_iam.PolicyStatement({ + actions: ['ssm:GetParametersByPath'], + resources: [`arn:aws:ssm:${this.region}:*:parameter/local/*`], + }), + ); + + fn.addToRolePolicy( + new cdk.aws_iam.PolicyStatement({ + actions: [ + 's3:PutObject', + 's3:PutObjectAcl', + 's3:GetObject', + 's3:GetObjectAcl', + 's3:DeleteObject', + ], + resources: [params.docsBucket.bucketArn], + }), + ); + } +} diff --git a/cdk/init-stack.ts b/cdk/init-stack.ts index 76bb6c7da..ce49d9934 100644 --- a/cdk/init-stack.ts +++ b/cdk/init-stack.ts @@ -22,6 +22,7 @@ import * as dotenv from 'dotenv'; import ContactCompleteStack from './contact-complete-stack'; import ContactCoreStack from './contact-core-stack'; import ContactRetrieveStack from './contact-retrieve-stack'; +import HrmSearchIndexStack from './hrm-search-index-stack'; import HrmMicoservicesStack from './hrm-micoroservices-stack'; import LocalCoreStack from './local-core-stack'; import ResourcesCoreStack from './resources-core-stack'; @@ -100,6 +101,18 @@ async function main() { }, }); + new HrmSearchIndexStack({ + scope: app, + id: 'hrm-search-index', + params: { + completeQueue: contactComplete.completeQueue, + docsBucket: localCore.docsBucket, + }, + props: { + env: { region: app.node.tryGetContext('region') }, + }, + }); + new ResourcesCoreStack({ scope: app, id: 'resources-core', diff --git a/hrm-domain/hrm-service/service-tests/case.test.ts b/hrm-domain/hrm-service/service-tests/case.test.ts index a67c9fb1b..2cc1201d5 100644 --- a/hrm-domain/hrm-service/service-tests/case.test.ts +++ b/hrm-domain/hrm-service/service-tests/case.test.ts @@ -29,7 +29,11 @@ import { CaseService } from '@tech-matters/hrm-core/case/caseService'; import * as caseDb from '@tech-matters/hrm-core/case/caseDataAccess'; import { convertCaseInfoToExpectedInfo } from './case/caseValidation'; -import { mockingProxy, mockSuccessfulTwilioAuthentication } from '@tech-matters/testing'; +import { + mockingProxy, + mockSuccessfulTwilioAuthentication, + newSQSmock, +} from '@tech-matters/testing'; import * as mocks from './mocks'; import { ruleFileActionOverride } from './permissions-overrides'; import { headers, getRequest, getServer, setRules, useOpenRules } from './server'; @@ -44,18 +48,35 @@ const request = getRequest(server); const { case1, case2, accountSid, workerSid } = mocks; -afterAll(done => { - mockingProxy.stop().finally(() => { - server.close(done); - }); -}); +let hrmIndexSQSMock: ReturnType; beforeAll(async () => { await mockingProxy.start(); + const mockttp = await mockingProxy.mockttpServer(); + hrmIndexSQSMock = newSQSmock({ + mockttp, + pathPattern: + /\/(test|local|development)\/xx-fake-1\/sqs\/jobs\/hrm-search-index\/queue-url-consumer/, + }); +}); + +afterAll(async () => { + await hrmIndexSQSMock.teardownSQSMock(); + await mockingProxy.stop(); + server.close(); }); beforeEach(async () => { await mockSuccessfulTwilioAuthentication(workerSid); + await hrmIndexSQSMock.createSQSMockQueue({ + queueName: 'test-hrm-search-index-consumer-pending', + }); +}); + +afterEach(async () => { + // await hrmIndexSQSMock.drestoySQSMockQueue({ + // queueUrl: hrmIndexSQSMock.getMockSQSQueueUrl(), + // }); }); // eslint-disable-next-line @typescript-eslint/no-shadow @@ -102,7 +123,7 @@ describe('/cases route', () => { expect(response.status).toBe(401); expect(response.body.error).toBe('Authorization failed'); }); - test('should return 200', async () => { + test.only('should return 200', async () => { const response = await request.post(route).set(headers).send(case1); expect(response.status).toBe(200); diff --git a/hrm-domain/hrm-service/setTestEnvVars.js b/hrm-domain/hrm-service/setTestEnvVars.js index fa0256685..b8321c32f 100644 --- a/hrm-domain/hrm-service/setTestEnvVars.js +++ b/hrm-domain/hrm-service/setTestEnvVars.js @@ -50,3 +50,4 @@ process.env.ENABLE_CLEANUP_JOBS = true; process.env.AWS_REGION = 'xx-fake-1'; process.env.AWS_ACCESS_KEY_ID = 'mock-access-key'; process.env.AWS_SECRET_ACCESS_KEY = 'mock-secret-key'; +process.env.LOCAL_SQS_PORT = '3010'; diff --git a/packages/testing/index.ts b/packages/testing/index.ts index 63dd9c52f..b3a7494cd 100644 --- a/packages/testing/index.ts +++ b/packages/testing/index.ts @@ -20,3 +20,4 @@ export * from './mockSsm'; import { start, stop, mockttpServer } from './mocking-proxy'; export const mockingProxy = { start, stop, mockttpServer }; import './expectToParseAsDate'; +export { newSQSmock } from './mockSQS'; diff --git a/packages/testing/mockSQS.ts b/packages/testing/mockSQS.ts new file mode 100644 index 000000000..b23009085 --- /dev/null +++ b/packages/testing/mockSQS.ts @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import sqslite from 'sqslite'; +import { SQS } from 'aws-sdk'; +import { Mockttp } from 'mockttp'; +import { mockSsmParameters } from './mockSsm'; + +export const newSQSmock = ({ + mockttp, + pathPattern, +}: { + mockttp: Mockttp; + pathPattern: RegExp; +}) => { + const sqsService = sqslite({}); + const sqsClient = new SQS({ + endpoint: `http://localhost:${process.env.LOCAL_SQS_PORT}`, + }); + + let mockSQSQueueUrl: URL; + + const initializeSQSMock = async () => { + await sqsService.listen({ port: parseInt(process.env.LOCAL_SQS_PORT!) }); + await mockSsmParameters(mockttp, [ + { + pathPattern, + // /\/(test|local|development)\/xx-fake-1\/sqs\/jobs\/hrm-resources-search\/queue-url-index/, + valueGenerator: () => mockSQSQueueUrl.toString(), + }, + ]); + }; + + const teardownSQSMock = async () => { + await sqsService.close(); + }; + + const createSQSMockQueue = async ({ queueName }: { queueName: string }) => { + const { QueueUrl } = await sqsClient + .createQueue({ + QueueName: queueName, + }) + .promise(); + mockSQSQueueUrl = new URL(QueueUrl!); + }; + + const drestoySQSMockQueue = async ({ queueUrl }: { queueUrl: URL }) => { + await sqsClient + .deleteQueue({ + QueueUrl: queueUrl.toString(), + }) + .promise(); + }; + + const getMockSQSQueueUrl = () => mockSQSQueueUrl; + + return { + initializeSQSMock, + teardownSQSMock, + createSQSMockQueue, + drestoySQSMockQueue, + getMockSQSQueueUrl, + }; +};