diff --git a/src/main/javascript/update_code_display.js b/src/main/javascript/update_code_display.js new file mode 100644 index 0000000000..b753f3ab79 --- /dev/null +++ b/src/main/javascript/update_code_display.js @@ -0,0 +1,265 @@ +// This is a stand alone node script to iterate through the modules +// and update all code displays that do not match an allowed value. +// (i.e., code displays that would not pass FHIR validation) +// The script also attempts to standardize when the same code is used in +// multiple modules, and to minimize the number of updates. +// Any invalid/unknown codes will be printed to console. +// +// Note this does require the `sync-fetch` library to be installed first. +// +// Example: +// npm install sync-fetch +// node update_code_display.js /path/to/synthea/src/main/resources/modules/ +// +// Tested on node v20.3.1 + +const fetch = require('sync-fetch') +const process = require('process'); +const fs = require('fs'); +const path = require('path'); + +const moduleDirPath = process.argv[2]; + +let moduleDir = fs.opendirSync(moduleDirPath); + +const codeInventory = {}; +// codeInventory[system][code] = set("display1", "display2", ...) + +const allCodes = []; + +const codeDictionary = {}; +// codeDictionary[system][code] = "display to use" + +// systems to skip because tx.fhir.org doesn't know about them +const SYSTEMS_TO_SKIP = ['NUBC', 'DICOM-DCM', 'DICOM-SOP']; + +// codes used in Synthea as placeholders that we know aren't real +const PLACEHOLDER_CODES = ['999999', + "99999-0", "99999-1", "99999-2", "99999-3", "99999-4", + "99999-5", "99999-6", "99999-7", "99999-8", "99999-9", + "99999-10", "99999-11", + 'X9999-0', 'X9999-1', 'X9999-2']; + +if (fs.existsSync('./code_dictionary.json')) { + const rawJSON = fs.readFileSync('./code_dictionary.json'); + const loadedDictionary = JSON.parse(rawJSON); + Object.assign(codeDictionary, loadedDictionary); + console.log("Using previously saved ./code_dictionary.json") +} else { + processFiles(moduleDir, moduleDirPath, inventoryAllCodes); + buildDictionary(); + fs.writeFileSync('./code_dictionary.json', JSON.stringify(codeDictionary, null, 2)); + console.log("Saved ./code_dictionary.json") + // re-open to re-iterate through + moduleDir = fs.opendirSync(moduleDirPath); +} + +processFiles(moduleDir, moduleDirPath, checkAndUpdateAllCodes); + +function processFiles(dirEntry, parentPath, fileFunction) { + let fileInFolder = dirEntry.readSync(); + while (fileInFolder != null) { + if(fileInFolder.isDirectory()) { + const newPath = path.join(parentPath, fileInFolder.name); + const subDir = fs.opendirSync(newPath); + processFiles(subDir, newPath, fileFunction); + } else if (fileInFolder.isFile() && fileInFolder.name.endsWith('.json')) { + const moduleJSONPath = path.join(parentPath, fileInFolder.name); + fileFunction(moduleJSONPath); + } + fileInFolder = dirEntry.readSync(); + } +} + +function inventoryAllCodes(moduleJSONPath) { + console.log(moduleJSONPath); + const rawJSON = fs.readFileSync(moduleJSONPath); + let module = JSON.parse(rawJSON); + + walkObject(module, inventoryCode); +} + +function checkAndUpdateAllCodes(moduleJSONPath) { + console.log(moduleJSONPath); + const rawJSON = fs.readFileSync(moduleJSONPath); + let module = JSON.parse(rawJSON); + + walkObject(module, checkAndUpdateCode); + + const updatedModuleJSON = JSON.stringify(module, null, 2); + fs.writeFileSync(moduleJSONPath, updatedModuleJSON); +} + +/** + * Recursively iterate through the given object and + * apply the given function to any "code" objects. + * A "code" object is one that has fields "system", "code", and "display". + */ +function walkObject(object, codeFunction) { + if (Array.isArray(object)) { + object.forEach(o => walkObject(o, codeFunction)); + } else if (typeof object === 'object' && object !== null) { + if (object.system && object.code && object.display) { + if (SYSTEMS_TO_SKIP.includes(object.system) || PLACEHOLDER_CODES.includes(object.code.toString())) { + return; + } + codeFunction(object); + } else { + for (const [key, value] of Object.entries(object)) { + walkObject(value, codeFunction); + } + } + } + // else, it's a null or primitive, nothing to do +} + +/** + * Add the given code to the inventory. + */ +function inventoryCode(codeObj) { + let { system, code, display } = codeObj; + code = code.toString(); + + if (!codeInventory[system]) { + codeInventory[system] = {}; + } + if (!codeInventory[system][code]) { + codeInventory[system][code] = new Set(); + allCodes.push(codeObj); + } + + codeInventory[system][code].add(display); +} + +function buildDictionary() { + for (const codeObj of allCodes) { + let { system, code, display } = codeObj; + code = code.toString(); + if (!codeDictionary[system]) { + codeDictionary[system] = {}; + } + + const body = requestBody(codeObj); + const VALIDATE_CODE_HEADERS = { + "Content-Type": "application/fhir+json", + "Accept": "application/fhir+json" + }; + const res = fetch( + 'http://tx.fhir.org/r4/CodeSystem/$validate-code?', + { + method: 'POST', + body: JSON.stringify(body, null, 2), + headers: VALIDATE_CODE_HEADERS + }).json(); + + const displayParam = res.parameter.find(p => p.name === 'display'); + const success = res.parameter.find(p => p.name === 'result').valueBoolean; + if (success) { + // the current display may or may not be the singular best, + // but it is one of the allowed values. + // the "display" param contains the official best value + const bestDisplay = displayParam.valueString; + + if (codeInventory[system][code].has(bestDisplay)) { + // we use the best display somewhere already, stick to it + codeDictionary[system][code] = bestDisplay; + } else { + // standardize to the current display + codeDictionary[system][code] = display; + } + } else { + if (displayParam) { + // the code is valid but the display is not allowed + // update the code to use the given display and + // store the given display going forward + codeDictionary[system][code] = displayParam.valueString; + } else { + handleErrorOrExit(body, res); + } + } + } +} + +/** + * Construct the request body to validate the given code. + */ +function requestBody(codeObj) { + const coding = JSON.parse(JSON.stringify(codeObj)); // simple copy + coding.code = coding.code.toString(); + switch (coding.system) { + case 'SNOMED-CT': + coding.system = 'http://snomed.info/sct'; + break; + case 'LOINC': + coding.system = 'http://loinc.org'; + break; + case 'RxNorm': + coding.system = 'http://www.nlm.nih.gov/research/umls/rxnorm'; + break; + case 'CVX': + coding.system = 'http://hl7.org/fhir/sid/cvx'; + break; + + default: + console.log(`Unexpected code system ${coding.system} may not be supported by tx.fhir.org`); + } + return { + "resourceType": "Parameters", + "parameter": [ + { + "name": "coding", + "valueCoding": coding + }, + { + "name": "default-to-latest-version", + "valueBoolean": true + } + ] + } +} + +/** + * Check whether the display on the given code is appropriate, + * and if not, update it. + */ +function checkAndUpdateCode(codeObj) { + let { system, code, display } = codeObj; + code = code.toString(); + if (!codeDictionary[system][code]) { + // something went wrong in validating the code earlier, + // the log will show what it was + return; + } + + // capitalization doesn't appear to matter, so only update a code + // if it differs in more than just caps + if (codeDictionary[system][code].toLowerCase() != display.toLowerCase()) { + console.log(`Updating "${display}" -> "${codeDictionary[system][code]}"`) + codeObj.display = codeDictionary[system][code]; + } +} + +/** + * Handle errors from the terminology service. + * "Unknown code" errors are expected so just print those out to the log. + * Anything else, print the request and response then exit. + */ +function handleErrorOrExit(body, res) { + const messageParam = res.parameter.find(p => p.name === 'message'); + if (messageParam) { + const message = messageParam.valueString; + + if (message.startsWith("Unable to find code ") + || message.startsWith("Unknown Code '")) { + console.log(message); + return; + } + } + // something else went wrong. print message and stop + console.log("Error occurred.") + console.log("Request body:") + console.log(JSON.stringify(body, null, 2)); + console.log("\nResponse:") + console.log(JSON.stringify(res, null, 2)); + process.exit(1); +}