From e266863c7fa836bc739b0cb52c98393b35df130a Mon Sep 17 00:00:00 2001 From: Anro Date: Wed, 6 Nov 2024 12:31:42 +0200 Subject: [PATCH] fix: prototype duplicate prevention --- webapp/package-lock.json | 27 ++ webapp/package.json | 2 + .../components/enketo/enketo.component.html | 5 + webapp/src/ts/main.ts | 17 ++ .../contacts/contacts-edit.component.ts | 249 +++++++++++++++++- webapp/src/ts/polyfills.ts | 40 +++ .../contacts/contacts-edit.component.spec.ts | 38 ++- 7 files changed, 375 insertions(+), 3 deletions(-) diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 98cddc8aa27..c733b2574e4 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -25,6 +25,7 @@ "@ngrx/effects": "^17.1.1", "@ngrx/store": "^17.1.1", "@ngx-translate/core": "^14.0.0", + "@types/levenshtein": "1.0.4", "@webcomponents/webcomponentsjs": "^2.8.0", "bikram-sambat-bootstrap": "^1.6.0", "bootstrap": "^3.4.1", @@ -34,6 +35,7 @@ "eurodigit": "^3.1.3", "font-awesome": "^4.7.0", "jquery": "3.5.1", + "levenshtein": "1.0.5", "lodash-es": "^4.17.21", "moment-locales-webpack-plugin": "^1.2.0", "ngrx-store-logger": "^0.2.4", @@ -4808,6 +4810,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/levenshtein": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/levenshtein/-/levenshtein-1.0.4.tgz", + "integrity": "sha512-QiNzDEGuAHoNVa7xjTPGQRecXScckE8bAEpuHipG8lEFPZh4eIBK0dw0K5mu9XdiTiVD8AxwYY8lOxYaP1rZUA==", + "license": "MIT" + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -8687,6 +8695,15 @@ "node": ">=0.10.0" } }, + "node_modules/levenshtein": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/levenshtein/-/levenshtein-1.0.5.tgz", + "integrity": "sha512-UQf1nnmxjl7O0+snDXj2YF2r74Gkya8ZpnegrUBYN9tikh2dtxV/ey8e07BO5wwo0i76yjOvbDhFHdcPEiH9aA==", + "engines": [ + "node >=0.2.0" + ], + "license": "Public Domain" + }, "node_modules/license-webpack-plugin": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/license-webpack-plugin/-/license-webpack-plugin-4.0.2.tgz", @@ -16839,6 +16856,11 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "@types/levenshtein": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/levenshtein/-/levenshtein-1.0.4.tgz", + "integrity": "sha512-QiNzDEGuAHoNVa7xjTPGQRecXScckE8bAEpuHipG8lEFPZh4eIBK0dw0K5mu9XdiTiVD8AxwYY8lOxYaP1rZUA==" + }, "@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -19648,6 +19670,11 @@ "integrity": "sha512-MYUxjSQSBUQmowc0l5nPieOYwMzGPUaTzB6inNW/bdPEG9zOL3eAAD1Qw5ZxSPk7we5dMojHwNODYMV1hq4EVg==", "dev": true }, + "levenshtein": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/levenshtein/-/levenshtein-1.0.5.tgz", + "integrity": "sha512-UQf1nnmxjl7O0+snDXj2YF2r74Gkya8ZpnegrUBYN9tikh2dtxV/ey8e07BO5wwo0i76yjOvbDhFHdcPEiH9aA==" + }, "license-webpack-plugin": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/license-webpack-plugin/-/license-webpack-plugin-4.0.2.tgz", diff --git a/webapp/package.json b/webapp/package.json index 7a272c60fe7..d4198387810 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -48,6 +48,8 @@ "eurodigit": "^3.1.3", "font-awesome": "^4.7.0", "jquery": "3.5.1", + "levenshtein": "1.0.5", + "@types/levenshtein": "1.0.4", "lodash-es": "^4.17.21", "moment-locales-webpack-plugin": "^1.2.0", "ngrx-store-logger": "^0.2.4", diff --git a/webapp/src/ts/components/enketo/enketo.component.html b/webapp/src/ts/components/enketo/enketo.component.html index 1a4f81d24b0..41ac0b3d931 100644 --- a/webapp/src/ts/components/enketo/enketo.component.html +++ b/webapp/src/ts/components/enketo/enketo.component.html @@ -1,5 +1,10 @@
+ +
diff --git a/webapp/src/ts/main.ts b/webapp/src/ts/main.ts index d4b492a22c9..7563d56dc97 100644 --- a/webapp/src/ts/main.ts +++ b/webapp/src/ts/main.ts @@ -7,6 +7,22 @@ window.startupTimes.firstCodeExecution = performance.now(); window.PouchDB = require('pouchdb-browser').default; window.$ = window.jQuery = require('jquery'); +window._phdcChanges = { + hierarchyDuplicatePrevention: { + health_center: { + ...Levenshtein, + props: [ + {form_prop_path: `/data/health_center/name`, db_doc_ref: 'name'}, + {form_prop_path: '/data/health_center/external_id', db_doc_ref: 'external_id'} + ], + queryParams: { + valuePaths: ['/data/health_center/is_user_flagged_duplicate', '/data/health_center/duplicate/action'], + // eslint-disable-next-line eqeqeq + query: (duplicate, action) => duplicate === 'yes' && action != null + } + } + } +}; import { enableProdMode } from '@angular/core'; import '@angular/compiler'; @@ -19,6 +35,7 @@ import { environment } from '@mm-environments/environment'; import { POUCHDB_OPTIONS } from './constants'; import * as bootstrapper from '../js/bootstrapper'; +import { Levenshtein } from './polyfills'; // Moment additional locales require('../js/moment-locales/tl'); diff --git a/webapp/src/ts/modules/contacts/contacts-edit.component.ts b/webapp/src/ts/modules/contacts/contacts-edit.component.ts index e0407b7caf4..fca0688249f 100644 --- a/webapp/src/ts/modules/contacts/contacts-edit.component.ts +++ b/webapp/src/ts/modules/contacts/contacts-edit.component.ts @@ -14,6 +14,22 @@ import { GlobalActions } from '@mm-actions/global'; import { PerformanceService } from '@mm-services/performance.service'; import { TranslateService } from '@mm-services/translate.service'; +import * as moment from 'moment'; +import * as Levenshtein from 'levenshtein'; +type HierarchyDuplicatePreventionType = typeof window._phdcChanges.hierarchyDuplicatePrevention; +type StrategyForContactType = HierarchyDuplicatePreventionType[T]; +type KeysType = keyof HierarchyDuplicatePreventionType; +type ConfType = StrategyForContactType; +type Strategy = Exclude; +type PropsToCheckType = Strategy['props']; +type QueryParamsInfoType = Strategy['queryParams']; +// Each record WILL have the following props, but could have more +type Sibling = {_id: string; name: string; reported_date: number; [key: string]: any}; +type Duplicate = {_id: string; name: string; reported_date: number; score: number}; +type SuccessType = { status: 'fulfilled'; value: any }; +type FailureType = { status: 'rejected'; reason: any }; +type ReturnType = SuccessType | FailureType; + @Component({ templateUrl: './contacts-edit.component.html' }) @@ -55,6 +71,22 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { private trackSave; private trackMetadata = { action: '', form: '' }; + private duplicateThreshold = 0; // Default is exact matches + private formPropPathsToCheck: PropsToCheckType = []; // Name is checked by default + private queryParams: QueryParamsInfoType; + private isEdit!: boolean; + private parentId!: string; + private contactType!: string; + private dbLookupRef; // sibling database request + private needsCleanup = false; + private readonly parser = new DOMParser(); // Used to parse the xml content & easily grab values + private readonly strategies = { + NormalizedLevenshtein: this.normalizedLevenshtein, + Levenshtein: this.levenshteinDistance + }; + + private strategy = this.levenshteinDistance; + ngOnInit() { this.trackRender = this.performanceService.track(); this.subscribeToStore(); @@ -171,6 +203,51 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { const formInstance = await this.renderForm(formId, titleKey); this.setEnketoContact(formInstance); + // When using the link to go to another form, we need to clean up the duplicate section + if (this.needsCleanup){ + this.cleanUpDuplicateSection(); + this.needsCleanup = false; + } + + const docId = this.enketoContact.docId; + // eslint-disable-next-line eqeqeq + this.isEdit = docId != null; + this.parentId = contact?.parent?._id ?? this.routeSnapshot.params.parent_id; + this.contactType = contactType?.id ?? this.enketoContact?.type; + + if (window?._phdcChanges?.hierarchyDuplicatePrevention){ + if (this.contactType in window._phdcChanges.hierarchyDuplicatePrevention){ + const ct: KeysType = (this.contactType as KeysType); + const conf: StrategyForContactType = window._phdcChanges.hierarchyDuplicatePrevention[ct]; + + if (conf){ + const strategyType = conf.type; + this.strategy = this.strategies[strategyType]; + this.formPropPathsToCheck = conf.props ? conf.props: + [{form_prop_path: `/data/${this.contactType}/name`, db_doc_ref: 'name'}]; + this.duplicateThreshold = conf.threshold; + this.queryParams = conf.queryParams; + } else { + console.error('No config has been loaded!'); + } + } else { + console.warn(`Setup omitted contact type "${this.contactType}" from duplicate check.`); + } + } else { + console.log('Namespace does not contain config'); + } + + // We're kicking off the sibling lookup in the form init to give the request a chance + // to make headway in the background before the submit function needs to process the results. + // The impact should only really be felt by bigger calls. + this.dbLookupRef = this.dbService + .get() + .query('medic-client/contacts_by_parent', { // Existing CHT view + startkey: [this.parentId, this.contactType], + endkey: [this.parentId, this.contactType, {}], + include_docs: true + }); + this.globalActions.setLoadingContent(false); } catch (error) { this.errorTranslationKey = error.translationKey || 'error.loading.form'; @@ -299,6 +376,152 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { }; } + private parseXmlForm(form): Document | undefined { + // Below line is taken from enketo-core form-model line 114 + const xmlDoc = this.parser.parseFromString(form.getDataStr({ irrelevant: false }), 'text/xml'); + if (xmlDoc.getElementsByTagName('parsererror').length > 0) { + console.error('Error parsing XML:', xmlDoc.getElementsByTagName('parsererror')[0].textContent); + this.globalActions.setEnketoError('Error parsing xml document'); + return; + } + return xmlDoc; + } + + private getFormValues(xmlDoc: Document, propPaths: string[]){ + const values:string[] = []; + for (const path of propPaths){ + try { + const value = xmlDoc.evaluate( + path, + xmlDoc, + null, + XPathResult.STRING_TYPE, + null + ).stringValue; + values.push(value); + } catch (error) { + console.error(`Path ${path} value could not be resolved!`); + } + } + return values; + } + + private queryPropInfo(xmlDoc: Document, info: Exclude){ + const propPaths = info.valuePaths; + const values = this.getFormValues(xmlDoc, propPaths); + const result = info.query(...values); + return result; + } + + private calculateSiblingLikenessScore(sibling: Sibling, props: { value: string; compareToDocProp: string }[]) { + let totalScore = 0; + let count = 0; + + for (const field of props) { + if (Object.prototype.hasOwnProperty.call(sibling, field.compareToDocProp)) { + const test = this.strategy(field.value.toLowerCase(), sibling[field.compareToDocProp].toLowerCase()); + totalScore += test; + count++; + } + } + + return count > 0 ? totalScore / count : null; + } + + private getLikelyDuplicates(siblings: Sibling[], props: {value: string; compareToDocProp: string}[]){ + const duplicates: Array = []; + for (const sibling of siblings){ + const score = this.calculateSiblingLikenessScore(sibling, props); + if (score !== null && score < this.duplicateThreshold){ + duplicates.push({_id: sibling._id, name: sibling.name, reported_date: sibling.reported_date, score}); + } + } + + return duplicates; + } + + // Normalize the distance by dividing by the length of the longer string. + // This can make the metric more adaptable across different string lengths + private normalizedLevenshtein(str1: string, str2: string) { + const distance = this.levenshteinDistance(str1, str2); + const maxLen = Math.max(str1.length, str2.length); + return (maxLen === 0) ? 0 : (distance / maxLen); + } + + // The Levenshtein distance is a measure of the number of edits (insertions, deletions, and substitutions) + // required to change one string into another. + private levenshteinDistance(str1: string, str2: string) { + return new Levenshtein(str1, str2).distance; + } + + // Note: Look into 'https://www.npmjs.com/package/fuzzball' to perform various matching calculations + + private cleanUpDuplicateSection() { + const $duplicateInfoElement = $('#contact-form').find('#duplicate_info'); + $duplicateInfoElement.empty(); // Remove all child nodes + $duplicateInfoElement.hide(); + } + + // Promise.allSettled is not available due to the app's javascript version + private allSettledFallback(promises: Promise>[]): Promise { + return Promise.all( + promises.map(promise => promise + .then((value): SuccessType => ({ status: 'fulfilled', value })) + .catch((reason): FailureType => ({ status: 'rejected', reason }))) + ); + } + + private buildDuplicateHeader(count: number){ + return `

${count} potential duplicate item(s) found:

`; + } + + private buildDuplicateItem(d: Duplicate, count: number){ + const link = `#/contacts/${d._id}/edit`; + const date = moment(d.reported_date).format('ddd MMM DD YYYY HH:mm:ss'); + return ` +

${count}) "${d.name}"
+ Created on: ${date}
+ Take me there +

`; + } + + private outputDuplicates(duplicates: Duplicate[]){ + const $duplicateInfoElement = $('#contact-form').find('#duplicate_info'); + $duplicateInfoElement.empty(); // Remove all child nodes + $duplicateInfoElement.show(); + // TODO: create a template component where these values are fed into. + // TODO: Use angular instead of jQuery? + let content = this.buildDuplicateHeader(duplicates.length); + let count = 0; + for (const d of duplicates){ + content += this.buildDuplicateItem(d, count+1); + count++; + } + $duplicateInfoElement.append(content); + $duplicateInfoElement.on('click', '.duplicate-navigate-link', () => { + this.needsCleanup = true; + }); + } + + private ensureUniqueItem(xmlDoc: Document, docId: string, docs: Sibling[]) { + const siblings : Sibling[] = docs.filter((element) => !((this.isEdit && element._id === docId))); + + const additionalFieldsValues:string[] = this.formPropPathsToCheck ? + this.getFormValues(xmlDoc, this.formPropPathsToCheck.map((e) => e.form_prop_path)) : []; + const additionalPropValuesToCheck = this.formPropPathsToCheck!.map((obj, i) => { + return { value: additionalFieldsValues[i], compareToDocProp: obj.db_doc_ref}; + }); + const duplicates = this.getLikelyDuplicates(siblings, additionalPropValuesToCheck); + + if (duplicates.length > 0){ + this.globalActions.setEnketoError('Duplicates found'); + this.outputDuplicates(duplicates); + return false; + } + + return true; + } + save() { if (this.enketoSaving) { console.debug('Attempted to call contacts-edit:save more than once'); @@ -315,8 +538,30 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { this.globalActions.setEnketoSavingStatus(true); this.globalActions.setEnketoError(null); - return Promise - .resolve(form.validate()) + const xmlDoc = this.parseXmlForm(form); + if (!xmlDoc){ + throw new Error('Could not parse form'); + } + + const shouldCircumventDuplicateCheck = this.queryParams ? this.queryPropInfo(xmlDoc, this.queryParams) : false; + + return this.allSettledFallback([form.validate(), this.dbLookupRef]) + .then((results) => { + let valid = results[0].status === 'fulfilled'? results[0].value : false; + + if (valid && !shouldCircumventDuplicateCheck) { + if (results[1].status === 'fulfilled'){ + const additionalCheckResult = results[1].value; + const map = additionalCheckResult.rows.map((row: { doc: Sibling }) => row.doc); + valid = this.ensureUniqueItem(xmlDoc, docId, map); + } else { + valid = false; + this.globalActions.setEnketoError('Sibling lookup failed'); + } + } + + return valid; + }) .then((valid) => { if (!valid) { throw new Error('Validation failed.'); diff --git a/webapp/src/ts/polyfills.ts b/webapp/src/ts/polyfills.ts index 30212e837d0..f9dc17d1878 100755 --- a/webapp/src/ts/polyfills.ts +++ b/webapp/src/ts/polyfills.ts @@ -88,7 +88,14 @@ declare global { CHTCore: any; angular: any; EnketoForm:any; + _phdcChanges: { // Additional namespace + // Specify your own contact_types here + hierarchyDuplicatePrevention: Partial<{[key in 'person' | 'health_center']: Strategy;}>; + // The Partial utility ensures that only the allowed keys (health_center, clinic, person, etc) are used, + // but none are mandatory. + }; } + interface JQuery { daterangepicker(options?: any, callback?: Function) : any; select2(event?:any, options?:any):any; @@ -98,6 +105,39 @@ declare global { } } +type Strategy = { + // Name is used by default (since it's hardcoded in the system) if no props are specified. + // Should props be specified, only those items will be considered + props?: {form_prop_path: string; db_doc_ref: string}[]; // TODO: perhaps add a weight to each property + queryParams?: { // Usage example - see main.ts + valuePaths: string[]; + //https://stackoverflow.com/questions/76130608/in-typescript-can-you-define-a-function-type-where-each-argument-can-be-one-of + query: (...valuesAtPaths: string[]) => boolean; + // TODO: ^ pass back tuple key & value ^ + }; +} & (LevenshteinType | NormalizedLevenshteinType); +// https://dev.to/darkmavis1980/what-are-typescript-discriminated-unions-5hbb + +type LevenshteinType = { + type: 'Levenshtein'; + threshold: number; +} + +type NormalizedLevenshteinType = { + type: 'NormalizedLevenshtein'; + threshold: number; // TODO: Enforce range between 0 and 1 here +} + +export const Levenshtein: Strategy = { + type: 'Levenshtein', + threshold: 1.5, +}; + +export const NormalizedLevenshtein: Strategy = { + type: 'NormalizedLevenshtein', + threshold: 0.334, +}; + (window as any).process = { env: { DEBUG: undefined }, browser: true, diff --git a/webapp/tests/karma/ts/modules/contacts/contacts-edit.component.spec.ts b/webapp/tests/karma/ts/modules/contacts/contacts-edit.component.spec.ts index 61c63b0d16c..cf6682c3baa 100644 --- a/webapp/tests/karma/ts/modules/contacts/contacts-edit.component.spec.ts +++ b/webapp/tests/karma/ts/modules/contacts/contacts-edit.component.spec.ts @@ -26,6 +26,7 @@ describe('ContactsEdit component', () => { let router; let route; let dbGet; + let viewLookup; let createComponent; let fixture; let component; @@ -34,6 +35,7 @@ describe('ContactsEdit component', () => { let routeSnapshot; let stopPerformanceTrackStub; let performanceService; + let dummyFormXml; beforeEach(() => { contactTypesService = { @@ -43,6 +45,7 @@ describe('ContactsEdit component', () => { }; translateService = { get: sinon.stub().resolvesArg(0) }; dbGet = sinon.stub().resolves(); + viewLookup = sinon.stub().resolves(); router = { navigate: sinon.stub() }; routeSnapshot = { params: {}, queryParams: {} }; route = { @@ -81,7 +84,7 @@ describe('ContactsEdit component', () => { providers: [ provideMockStore({ selectors: mockedSelectors }), { provide: TranslateService, useValue: translateService }, - { provide: DbService, useValue: { get: () => ({ get: dbGet }) } }, + { provide: DbService, useValue: { get: () => ({ get: dbGet, query: viewLookup }) } }, { provide: Router, useValue: router }, { provide: ActivatedRoute, useValue: route }, { provide: LineageModelGeneratorService, useValue: lineageModelGeneratorService }, @@ -99,9 +102,16 @@ describe('ContactsEdit component', () => { return TestBed.compileComponents().then(() => { fixture = TestBed.createComponent(ContactsEditComponent); component = fixture.componentInstance; + component.dbLookupRef = Promise.resolve({ + total_rows: 0, + offset: 0, + rows: [] + }); fixture.detectChanges(); }); }; + + dummyFormXml = `test`; }); afterEach(() => sinon.restore()); @@ -635,6 +645,7 @@ describe('ContactsEdit component', () => { component.enketoContact = { formInstance: { validate: sinon.stub().resolves(false), + getDataStr: sinon.stub().returns(dummyFormXml) }, }; @@ -654,6 +665,7 @@ describe('ContactsEdit component', () => { component.enketoContact = { formInstance: { validate: sinon.stub().resolves(true), + getDataStr: sinon.stub().returns(dummyFormXml) }, type: 'some_contact', }; @@ -680,6 +692,7 @@ describe('ContactsEdit component', () => { dbGet.resolves({ _id: 'clinic_create_form_id', the: 'form' }); const form = { validate: sinon.stub().resolves(true), + getDataStr: sinon.stub().returns(dummyFormXml) }; formService.render.resolves(form); @@ -687,6 +700,13 @@ describe('ContactsEdit component', () => { await fixture.whenStable(); formService.saveContact.resolves({ docId: 'new_clinic_id' }); + + // TODO: figure out why this test's dbLookupRef is null despite being set in the beforeEach + component.dbLookupRef = Promise.resolve({ + total_rows: 0, + offset: 0, + rows: [] + }); await component.save(); expect(performanceService.track.calledThrice).to.be.true; @@ -730,12 +750,20 @@ describe('ContactsEdit component', () => { dbGet.resolves({ _id: 'person_edit_form_id', the: 'form' }); const form = { validate: sinon.stub().resolves(true), + getDataStr: sinon.stub().returns(dummyFormXml) }; formService.render.resolves(form); await createComponent(); await fixture.whenStable(); + // TODO: figure out why this test's dbLookupRef is null despite being set in the beforeEach + component.dbLookupRef = Promise.resolve({ + total_rows: 0, + offset: 0, + rows: [] + }); + formService.saveContact.resolves({ docId: 'the_person' }); await component.save(); @@ -778,12 +806,20 @@ describe('ContactsEdit component', () => { dbGet.resolves({ _id: 'patient_create_form_id', the: 'form' }); const form = { validate: sinon.stub().resolves(true), + getDataStr: sinon.stub().returns(dummyFormXml) }; formService.render.resolves(form); await createComponent(); await fixture.whenStable(); + // TODO: figure out why this test's dbLookupRef is null despite being set in the beforeEach + component.dbLookupRef = Promise.resolve({ + total_rows: 0, + offset: 0, + rows: [] + }); + formService.saveContact.resolves({ docId: 'the_patient' }); await component.save();