Skip to content

Commit

Permalink
Merge pull request #119 from bcgsc/release/v6.4.0
Browse files Browse the repository at this point in the history 
Release/v6.4.0
  • Loading branch information
@mathieulemieux
mathieulemieux authored Mar 23, 2023
2 parents f8a4f8b + 2b2f29f commit 98d7797
Show file tree
Hide file tree
Showing 7 changed files with 584 additions and 97 deletions.
4 changes: 2 additions & 2 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@bcgsc-pori/graphkb-loader",
"main": "src/index.js",
"version": "6.3.3",
"version": "6.4.0",
"repository": {
"type": "git",
"url": "https://github.com/bcgsc/pori_graphkb_loader.git"
Expand Down Expand Up @@ -64,4 +64,4 @@
"start:signatures": "node bin/load.js file ontology data/signatures.json",
"start:vocabulary": "node bin/load.js file ontology data/vocab.json"
}
}
}
5 changes: 5 additions & 0 deletions src/civic/evidenceItems.graphql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@ query evidenceItems(
molecularProfile {
id
name
parsedName {
... on Gene { entrezId }
... on MolecularProfileTextSegment { text }
... on Variant { id }
}
rawName
variants {
gene {
Expand Down
164 changes: 71 additions & 93 deletions src/civic/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const { civic: SOURCE_DEFN, ncit: NCIT_SOURCE_DEFN } = require('../sources');
const { processVariantRecord } = require('./variant');
const { getRelevance } = require('./relevance');
const { getPublication } = require('./publication');
const { MolecularProfile } = require('./profile');
const { EvidenceItem: evidenceSpec } = require('./specs.json');

class NotImplementedError extends ErrorMixin { }
Expand Down Expand Up @@ -126,7 +127,7 @@ const getTherapy = async (conn, therapyRecord) => {
};


/** *
/**
* Add or fetch a therapy combination if there is not an existing record
* Link the therapy combination to its individual elements
*/
Expand Down Expand Up @@ -161,6 +162,9 @@ const addOrFetchTherapy = async (conn, source, therapiesRecords, combinationType
};


/**
* Add or fetch an evidence level if there is not an existing record
*/
const getEvidenceLevel = async ({
conn, rawRecord, sources,
}) => {
Expand Down Expand Up @@ -309,6 +313,10 @@ const processEvidenceRecord = async (opt) => {
content.subject = rid(await conn.getVocabularyTerm('patient'));
} if (rawRecord.evidenceType === 'FUNCTIONAL') {
content.subject = rid(feature);
} if (rawRecord.evidenceType === 'ONCOGENIC') {
content.subject = variants.length === 1
? rid(variants[0])
: rid(feature);
}

if (content.subject && !content.conditions.includes(content.subject)) {
Expand Down Expand Up @@ -405,13 +413,13 @@ const fetchDeletedEvidenceItems = async (url) => {
logger.info(`loading rejected evidenceItems from ${url}`);
const rejected = await requestEvidenceItems(url, {
query: `query evidenceItems($after: String, $status: EvidenceStatusFilter) {
evidenceItems(after: $after, status: $status) {
nodes {id}
pageCount
pageInfo {endCursor, hasNextPage}
totalCount
}
}`,
evidenceItems(after: $after, status: $status) {
nodes {id}
pageCount
pageInfo {endCursor, hasNextPage}
totalCount
}
}`,
variables: {
status: 'REJECTED',
},
Expand Down Expand Up @@ -476,46 +484,12 @@ const downloadEvidenceRecords = async (url, trustedCurators) => {
counts.error++;
continue;
}

if (
record.significance === 'NA'
|| (record.significance === null && record.evidenceType === 'PREDICTIVE')
) {
counts.skip++;
logger.debug(`skipping uninformative record (${record.id})`);
} else {
records.push(record);
}
records.push(record);
}
return { counts, errorList, records };
};


/**
* Splits a variant into a list of it's variations
* Desambiguate variants that as been linked as "or"
* Ex. {name: 'Q157P/R'} --> [{name: 'Q157P'}, {name: 'Q157R'}]
* Ex. {name: 'Q157P'} --> [{name: 'Q157P'}]
*
* @param {Object} variant the Variant object to desambigaute
* @returns {Object[]} an array of Variant objects
*/
const disambiguateVariant = (variant) => {
let variants = [variant],
orCombination;

if (orCombination = /^([a-z]\d+)([a-z])\/([a-z])$/i.exec(variant.name)) {
const [, prefix, tail1, tail2] = orCombination;
variants = [
{ ...variant, name: `${prefix}${tail1}` },
{ ...variant, name: `${prefix}${tail2}` },
];
}

return variants;
};


/**
* Access the CIVic API, parse content, transform and load into GraphKB
*
Expand Down Expand Up @@ -553,9 +527,8 @@ const upload = async ({
records: {},
};

// Refactor records into recordsById and varById
// Refactor records into recordsById
const recordsById = {};
const varById = {};

for (const record of records) {
// Check if max records limit has been reached
Expand All @@ -566,25 +539,22 @@ const upload = async ({

// Check if record id is unique
if (recordsById[record.id]) {
logger.warn(`Multiple evidenceItems with the same id: ${record.id}. Violates assumptions. Only the 1st one was kept.`);
logger.error(`Multiple evidenceItems with the same id: ${record.id}. Violates assumptions. Only the 1st one was kept.`);
counts.skip++;
continue;
}

// Introducing Molecular Profiles with CIViC GraphQL API v2.2.0
// [EvidenceItem]--(many-to-one)--[MolecularProfile]--(many-to-many)--[Variant]
if (record.molecularProfile && record.molecularProfile.id) {
if (record.molecularProfile.variants.length === 0) {
throw new Error(`Molecular Profile without Variant. Violates assumptions: ${record.molecularProfile.id}`);
} else if (record.molecularProfile.variants.length > 1) {
// TODO: Add support for Evidence Item with complex Molecular Profile
logger.warn(`Skip upload of Evidence Item with complex Molecular Profile (those with more that 1 Variant): ${record.id}`);
continue;
} else {
// Assuming 1 Variant per Molecular Profile
varById[record.molecularProfile.variants[0].id.toString()] = record.molecularProfile.variants[0];
}
} else {
throw new Error(`Evidence Item without Molecular Profile. Violates assumptions: ${record.id}`);
if (!record.molecularProfile) {
logger.error(`Evidence Item without Molecular Profile. Violates assumptions: ${record.id}`);
counts.skip++;
continue;
}
if (!record.molecularProfile.variants || record.molecularProfile.variants.length === 0) {
logger.error(`Molecular Profile without Variants. Violates assumptions: ${record.molecularProfile.id}`);
counts.skip++;
continue;
}

// Adding EvidenceItem to object for upload
Expand Down Expand Up @@ -626,55 +596,64 @@ const upload = async ({
continue;
}

// Variant disambiguation
// Assuming 1 Variant per Molecular Profile
record.variants = disambiguateVariant(varById[record.molecularProfile.variants[0].id.toString()]);
// Process Molecular Profiles expression into an array of conditions
// Each condition is itself an array of variants, one array for each expected GraphKB Statement from this CIViC Evidence Item
const Mp = MolecularProfile(record.molecularProfile);

try {
record.conditions = Mp.process().conditions;
} catch (err) {
logger.error(`evidence (${record.id}) ${err}`);
counts.skip += 1;
continue;
}

const oneToOne = (record.variants.length * record.therapies.length) === 1 && preupload.size === 1;
const postupload = [];

// Upload all GraphKB statements for this CIViC Evidence Item
for (const variant of record.variants) {
for (const therapies of record.therapies) {
try {
logger.debug(`processing ${record.id}`);
const result = await processEvidenceRecord({
conn,
oneToOne,
rawRecord: { ..._.omit(record, ['therapies', 'variants']), therapies, variant },
sources: { civic: source },
variantsCache,
});
postupload.push(rid(result));
counts.success += 1;
} catch (err) {
if (
err.toString().includes('is not a function')
|| err.toString().includes('of undefined')
) {
console.error(err);
for (const condition of record.conditions) {
const oneToOne = (condition.length * record.therapies.length) === 1 && preupload.size === 1;

for (const variant of condition) {
for (const therapies of record.therapies) {
try {
logger.debug(`processing ${record.id}`);
const result = await processEvidenceRecord({
conn,
oneToOne,
rawRecord: { ..._.omit(record, ['therapies', 'variants']), therapies, variant },
sources: { civic: source },
variantsCache,
});
postupload.push(rid(result));
counts.success += 1;
} catch (err) {
if (
err.toString().includes('is not a function')
|| err.toString().includes('of undefined')
) {
console.error(err);
}
if (err instanceof NotImplementedError) {
// accepted evidence that we do not support loading. Should delete as it may have changed from something we did support
purgeableEvidenceItems.add(sourceId);
}
errorList.push({ error: err, errorMessage: err.toString(), record });
logger.error(`evidence (${record.id}) ${err}`);
counts.error += 1;
}
if (err instanceof NotImplementedError) {
// accepted evidence that we do not support loading. Should delete as it may have changed from something we did support
purgeableEvidenceItems.add(sourceId);
}
errorList.push({ error: err, errorMessage: err.toString(), record });
logger.error(`evidence (${record.id}) ${err}`);
counts.error += 1;
}
}
}

// compare statments before/after upload to determine if any records should be soft-deleted
postupload.forEach((id) => {
preupload.delete(id);
});

if (preupload.size && purgeableEvidenceItems.has(sourceId)) {
logger.warn(`
Removing ${preupload.size} CIViC Entries (EID:${sourceId}) of unsupported format
`);
Removing ${preupload.size} CIViC Entries (EID:${sourceId}) of unsupported format
`);

try {
await Promise.all(
Expand Down Expand Up @@ -733,7 +712,6 @@ const upload = async ({

module.exports = {
SOURCE_DEFN,
disambiguateVariant,
specs: { validateEvidenceSpec },
upload,
};
Loading

0 comments on commit 98d7797

Please sign in to comment.