Merge pull request #148 from bcgsc/Release/v8.0.0

Release/v8.0.0

Snakefile

@@ -167,14 +167,6 @@ rule download_cancerhotspots:
         ''')
 
 
-rule download_clinicaltrialsgov:
-    output: directory(f'{DATA_DIR}/clinicaltrialsgov')
-    shell: dedent(f'''\
-        mkdir -p {DATA_DIR}/clinicaltrialsgov
-        cd {DATA_DIR}/clinicaltrialsgov
-        wget https://clinicaltrials.gov/AllPublicXML.zip
-        unzip AllPublicXML.zip''')
-
 
 rule download_cosmic_resistance:
     output: f'{DATA_DIR}/cosmic/CosmicResistanceMutations.tsv'
@@ -413,12 +405,11 @@ rule load_approvals:
 rule load_clinicaltrialsgov:
     input: expand(rules.load_local.output, local=['vocab']),
         rules.all_diseases.output,
-        rules.all_drugs.output,
-        data=rules.download_clinicaltrialsgov.output
+        rules.all_drugs.output
     container: CONTAINER
     log: f'{LOGS_DIR}/clinicaltrialsgov.logs.txt'
     output: f'{DATA_DIR}/clinicaltrialsgov.COMPLETE'
-    shell: LOADER_COMMAND + ' api clinicaltrialsgov &> {log}; cp {log} {output}'
+    shell: LOADER_COMMAND + ' clinicaltrialsgov &> {log}; cp {log} {output}'
 
 
 rule load_cosmic_resistance:

bin/load.js

@@ -36,7 +36,7 @@ const cosmicResistance = require('../src/cosmic/resistance');
 const cosmicFusions = require('../src/cosmic/fusions');
 
 const API_MODULES = {
-    asco, clinicaltrialsgov, dgidb, docm, fdaApprovals, moa, oncotree
+    asco, dgidb, docm, fdaApprovals, moa, oncotree,
 };
 
 const FILE_MODULES = {
@@ -45,7 +45,6 @@ const FILE_MODULES = {
     cancerhotspots,
     cgi,
     cgl,
-    clinicaltrialsgov,
     diseaseOntology,
     drugbank,
     ensembl,
@@ -54,8 +53,8 @@ const FILE_MODULES = {
     ncit,
     ncitFdaXref,
     ontology,
-    sources,
     refseq,
+    sources,
     uberon,
     variants,
 };
@@ -70,11 +69,12 @@ const ALL_MODULES = {
     ...FILE_MODULES,
     ...COSMIC_MODULES,
     civic,
+    clinicaltrialsgov,
 };
 
 const parser = createOptionsMenu();
 
-const subparsers = parser.add_subparsers({ help: 'Sub-command help', required: true });
+const subparsers = parser.add_subparsers({ dest: 'subparser_name', help: 'Sub-command help', required: true });
 const apiParser = subparsers.add_parser('api');
 apiParser.add_argument('module', {
     choices: Object.keys(API_MODULES),
@@ -103,6 +103,11 @@ civicParser.add_argument('--trustedCurators', {
     nargs: '+',
 });
 
+const clinicaltrialsgovParser = subparsers.add_parser('clinicaltrialsgov');
+clinicaltrialsgovParser.add_argument('--days', {
+    help: 'Load new and existing studies added or modified (last update posted) in the last # of days',
+    type: Number,
+});
 
 const cosmicParser = subparsers.add_parser('cosmic');
 cosmicParser.add_argument('module', {
@@ -118,36 +123,24 @@ cosmicParser.add_argument('classification', {
     type: fileExists,
 });
 
-const { module: moduleName, input, ...options } = parser.parse_args();
+const {
+    subparser_name, module: moduleName, input, ...options
+} = parser.parse_args();
 
 let loaderFunction;
 
 if (input) {
-    loaderFunction = ALL_MODULES[moduleName || 'civic'].uploadFile;
+    loaderFunction = ALL_MODULES[moduleName || subparser_name].uploadFile;
 } else {
     debugger;
-    loaderFunction = ALL_MODULES[moduleName || 'civic'].upload;
+    loaderFunction = ALL_MODULES[moduleName || subparser_name].upload;
 }
 
 const loaderOptions = { ...options };
 
 if (input) {
     debugger;
-    if (moduleName === 'clinicaltrialsgov') {
-        if (fs.lstatSync(input).isDirectory()) {
-            const files = fs.readdirSync(input)
-                .map(filename => path.join(input, filename));
-            loaderOptions.files = files;
-        } else {
-            loaderOptions.files = [input];
-        }
-    } else {
-        loaderOptions.filename = input;
-
-        if (options.module === 'cosmic') {
-            loaderOptions.classification = options.classification;
-        }
-    }
+    loaderOptions.filename = input;
 }
 
 runLoader(options, loaderFunction, loaderOptions)

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bcgsc-pori/graphkb-loader",
   "main": "src/index.js",
-  "version": "7.0.1",
+  "version": "8.0.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/bcgsc/pori_graphkb_loader.git"
@@ -57,7 +57,7 @@
     "lint": "eslint -c .eslintrc.json src test",
     "version": "echo $npm_package_version",
     "start": "node bin/load.js",
-    "start:recent-trials": "node bin/load.js api clinicaltrialsgov",
+    "start:recent-trials": "node bin/load.js clinicaltrialsgov --days 14",
     "start:fda-approvals": "node bin/load.js api fdaApprovals",
     "start:oncotree": "node bin/load.js api oncotree",
     "start:civic": "node bin/load.js civic",

src/clinicaltrialsgov/README.md

@@ -5,8 +5,14 @@ This module loads clinical trials data into GraphKB from [https://www.clinicaltr
 > :warning: Since this loader produces statements, ontology and vocabulary data should be loaded first
 
 
-Uses REST API to load clinical trials data
+Uses REST API to load clinical trials data.
+```bash
+node bin/load.js clinicaltrialsgov
+```
 
+By default this loader loads all studies that related to cancer, which will be a huge number of records.
+Using `--days` can load the new and existing studies added or modified (last update posted) in the last # of days.
 ```bash
-node bin/load.js api clinicaltrialsgov
+node bin/load.js clinicaltrialsgov --days 7
 ```
+Loading the studies added or modified in the last week.