From bd92c146ef88ba9636b31b29f821a376e9d0a84b Mon Sep 17 00:00:00 2001 From: Seb Bacon Date: Mon, 13 May 2019 16:00:13 +0100 Subject: [PATCH] Use package provided by `converter` repo to transform data to CSV --- .../frontend/management/commands/load_data.py | 654 +----------------- requirements.in | 3 +- requirements.txt | 62 +- 3 files changed, 53 insertions(+), 666 deletions(-) diff --git a/clinicaltrials/frontend/management/commands/load_data.py b/clinicaltrials/frontend/management/commands/load_data.py index bcaa70b..1293286 100644 --- a/clinicaltrials/frontend/management/commands/load_data.py +++ b/clinicaltrials/frontend/management/commands/load_data.py @@ -1,95 +1,12 @@ # -*- coding: utf-8 -*- import logging -import sys -import traceback - -from bigquery import Client -from bigquery import StorageClient -from bigquery import TableExporter -from bigquery import wait_for_job -from bigquery import gen_job_name -import xmltodict import os -import subprocess -import json -import glob -import gzip -import datetime -import tempfile -import shutil import requests -import contextlib -import os -from bs4 import BeautifulSoup -import xmltodict -import json -from datetime import date -from datetime import datetime -from datetime import timedelta -from dateutil.relativedelta import relativedelta -import csv -import re -from google.cloud.exceptions import NotFound -from xml.parsers.expat import ExpatError - +import sys +import traceback +from frontend.management.commands.process_data import Command as ProcessCommand from django.core.management.base import BaseCommand -from django.conf import settings - - -logger = logging.getLogger(__name__) - -def raw_json_name(): - date = datetime.now().strftime('%Y-%m-%d') - return "raw_clincialtrials_json_{}.csv".format(date) - - -def postprocessor(path, key, value): - """Convert key names to something bigquery compatible - """ - if key.startswith('#') or key.startswith('@'): - key = key[1:] - if key == 'clinical_results': - # Arbitrarily long field that we don't need, see #179 - value = {'truncated_by_postprocessor': True} - return key, value - - -def wget_file(target, url): - subprocess.check_call(["wget", "-q", "-O", target, url]) - - -def download_and_extract(): - """Clean up from past runs, then download into a temp location and move the - result into place. - """ - logger.info("Downloading. This takes at least 30 mins on a fast connection!") - url = 'https://clinicaltrials.gov/AllPublicXML.zip' - - # download and extract - container = tempfile.mkdtemp( - prefix=settings.STORAGE_PREFIX.rstrip(os.sep), dir=settings.WORKING_VOLUME) - try: - data_file = os.path.join(container, "data.zip") - wget_file(data_file, url) - # Can't "wget|unzip" in a pipe because zipfiles have index at end of file. - with contextlib.suppress(OSError): - shutil.rmtree(settings.WORKING_DIR) - subprocess.check_call(["unzip", "-q", "-o", "-d", settings.WORKING_DIR, data_file]) - finally: - shutil.rmtree(container) - - -def upload_to_cloud(): - # XXX we should periodically delete old ones of these - logger.info("Uploading to cloud") - client = StorageClient() - bucket = client.get_bucket() - blob = bucket.blob( - "{}{}".format(settings.STORAGE_PREFIX, raw_json_name()), - chunk_size=1024*1024 - ) - with open(os.path.join(settings.WORKING_DIR, raw_json_name()), 'rb') as f: - blob.upload_from_file(f) +from ctconvert import create_instance def notify_slack(message): @@ -109,563 +26,30 @@ def notify_slack(message): ) -def convert_to_json(): - logger.info("Converting to JSON...") - dpath = os.path.join(settings.WORKING_DIR, 'NCT*/') - files = [x for x in sorted(glob.glob(dpath + '*.xml'))] - start = datetime.now() - completed = 0 - with open(os.path.join(settings.WORKING_DIR, raw_json_name()), 'w') as f2: - for source in files: - logger.info("Converting %s", source) - with open(source, 'rb') as f: - try: - f2.write( - json.dumps( - xmltodict.parse( - f, - item_depth=0, - postprocessor=postprocessor) - ) + "\n") - except ExpatError: - logger.warn("Unable to parse %s", source) - - completed += 1 - if completed % 100 == 0: - elapsed = datetime.now() - start - per_file = elapsed.seconds / completed - remaining = int(per_file * (len(files) - completed) / 60.0) - logger.info("%s minutes remaining", remaining) - - - -def convert_and_download(): - logger.info("Executing SQL in cloud and downloading results...") - storage_path = os.path.join(settings.STORAGE_PREFIX, raw_json_name()) - schema = [ - {'name': 'json', 'type': 'string'}, - ] - client = Client('clinicaltrials') - tmp_client = Client('tmp_eu') - table_name = settings.PROCESSING_STORAGE_TABLE_NAME - tmp_table = tmp_client.dataset.table("clincialtrials_tmp_{}".format(gen_job_name())) - with contextlib.suppress(NotFound): - table = client.get_table(table_name) - table.gcbq_table.delete() - - table = client.create_storage_backed_table( - table_name, - schema, - storage_path - ) - sql_path = os.path.join( - settings.BASE_DIR, 'frontend/view.sql') - with open(sql_path, 'r') as sql_file: - job = table.gcbq_client.run_async_query( - gen_job_name(), sql_file.read().format(table_name=table_name)) - job.destination = tmp_table - job.use_legacy_sql = False - job.write_disposition = 'WRITE_TRUNCATE' - job.begin() - - # The call to .run_async_query() might return before results are actually ready. - # See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#timeoutMs - wait_for_job(job) - - - t1_exporter = TableExporter(tmp_table, settings.STORAGE_PREFIX + 'test_table-') - t1_exporter.export_to_storage() - - with open(settings.INTERMEDIATE_CSV_PATH, 'w') as f: - t1_exporter.download_from_storage_and_unzip(f) - - -def get_env(path): - env = os.environ.copy() - with open(path) as e: - for k, v in re.findall(r"^export ([A-Z][A-Z0-9_]*)=(\S*)", e.read(), re.MULTILINE): - env[k] = v - return env +def convert_data(): + # This blocks until the compute instance stops running, and raises + # an exception if its startup script finished in an error or + # unknown state + create_instance.main( + "ebmdatalab", "europe-west2-a", "ctgov-converter", wait=True) def process_data(): - # TODO no need to call via shell any more (now we are also a command) - try: - subprocess.check_output( - [ - "{}python".format(settings.PROCESSING_VENV_BIN), - "{}/manage.py".format(settings.BASE_DIR), - "process_data", - "--input-csv={}".format(settings.INTERMEDIATE_CSV_PATH), - "--settings=frontend.settings" - ], - stderr=subprocess.STDOUT, - env=get_env(settings.PROCESSING_ENV_PATH)) - notify_slack("Today's data uploaded to FDAAA staging: " - "https://staging-fdaaa.ebmdatalab.net. " - "If this looks good, tell ebmbot to " - "'@ebmbot fdaaa deploy'""") - except subprocess.CalledProcessError as e: - notify_slack("Error in FDAAA import: command `{}` " - "failed with error code {} " - "and output {}".format( - e.cmd, e.returncode, e.output)) - sys.exit(1) - - -################################### -# Helper functions for CSV assenbly -################################### - -def is_covered_phase(phase): - return phase in [ - "Phase 1/Phase 2", - "Phase 2", - "Phase 2/Phase 3", - "Phase 3", - "Phase 4", - "N/A", - ] - - -def is_not_withdrawn(study_status): - return study_status != "Withdrawn" - - -def is_interventional(study_type): - return study_type == "Interventional" - - -def is_covered_intervention(intervention_type_list): - covered_intervention_type = [ - "Drug", - "Device", - "Biological", - "Genetic", - "Radiation", - "Combination Prodcut", - "Diagnostic Test", - ] - a_set = set(covered_intervention_type) - b_set = set(intervention_type_list) - if a_set & b_set: - return True - else: - return False - - -def is_not_device_feasibility(primary_purpose): - return primary_purpose != "Device Feasibility" - - -def is_fda_reg(fda_reg_drug, fda_reg_device): - if fda_reg_drug == "Yes" or fda_reg_device == "Yes": - return True - else: - return False - - -def is_old_fda_regulated(is_fda_regulated, fda_reg_drug, fda_reg_device): - if ( - fda_reg_drug is None and fda_reg_device is None - ) and is_fda_regulated is not False: - return True - else: - return False - - -def has_us_loc(locs): - us_locs = [ - "United States", - "American Samoa", - "Guam", - "Northern Mariana Islands", - "Puerto Rico", - "Virgin Islands (U.S.)", - ] - if locs: - for us_loc in us_locs: - if us_loc in locs: - return True - return False - - -def dict_or_none(data, keys): - for k in keys: - try: - data = data[k] - except KeyError: - return None - return json.dumps(data, separators=(',', ':')) - - -# Some dates on clinicaltrials.gov are only Month-Year not -# Day-Month-Year. When this happens, we assign them to the last day -# of the month so our "results due" assessments are conservative -def str_to_date(datestr): - is_defaulted_date = False - if datestr is not None: - try: - parsed_date = datetime.strptime(datestr.text, "%B %d, %Y").date() - except ValueError: - parsed_date = ( - datetime.strptime(datestr.text, "%B %Y").date() - + relativedelta(months=+1) - - timedelta(days=1) - ) - is_defaulted_date = True - else: - parsed_date = None - return (parsed_date, is_defaulted_date) - - -def t(textish): - if textish is None: - return None - return textish.text - - -def does_it_exist(dataloc): - if dataloc is None: - return False - else: - return True - - -def convert_bools_to_ints(row): - for k, v in row.items(): - if v is True: - v = 1 - row[k] = v - elif v is False: - v = 0 - row[k] = v - return row - - -def convert_to_csv(): - headers = [ - "nct_id", - "act_flag", - "included_pact_flag", - "has_results", - "pending_results", - "pending_data", - "has_certificate", - "results_due", - "start_date", - "available_completion_date", - "used_primary_completion_date", - "defaulted_pcd_flag", - "defaulted_cd_flag", - "results_submitted_date", - "last_updated_date", - "certificate_date", - "phase", - "enrollment", - "location", - "study_status", - "study_type", - "primary_purpose", - "sponsor", - "sponsor_type", - "collaborators", - "exported", - "fda_reg_drug", - "fda_reg_device", - "is_fda_regulated", - "url", - "title", - "official_title", - "brief_title", - "discrep_date_status", - "late_cert", - "defaulted_date", - "condition", - "condition_mesh", - "intervention", - "intervention_mesh", - "keywords", - ] - - cs = "clinical_study" - effective_date = date(2017, 1, 18) - - # This is a snapshot of CT.gov at a time when it included FDA - # regulation metadata - fda_reg_dict = {} - with gzip.open( - os.path.join(settings.BASE_DIR, - 'frontend', 'management', 'commands', - 'fdaaa_regulatory_snapshot.csv.gz'), 'rt') as old_fda_reg: - reader = csv.DictReader(old_fda_reg) - for d in reader: - fda_reg_dict[d["nct_id"]] = d["is_fda_regulated"] - - - with open(settings.INTERMEDIATE_CSV_PATH, 'w', newline="", encoding="utf-8") as test_csv: - writer = csv.DictWriter(test_csv, fieldnames=headers) - writer.writeheader() - dpath = os.path.join(settings.WORKING_DIR, 'NCT*/') - files = [x for x in sorted(glob.glob(dpath + '*.xml'))] - - for xml_filename in files: - with open(xml_filename) as raw_xml: - soup = BeautifulSoup(raw_xml, "xml") - with open(xml_filename) as xml_to_json: - parsed_json = xmltodict.parse(xml_to_json.read()) - - td = {} - - td["nct_id"] = t(soup.nct_id) - - td["study_type"] = t(soup.study_type) - - td["has_certificate"] = does_it_exist(soup.disposition_first_submitted) - - td["phase"] = t(soup.phase) - - td["fda_reg_drug"] = t(soup.is_fda_regulated_drug) - - td["fda_reg_device"] = t(soup.is_fda_regulated_device) - - td["primary_purpose"] = t(soup.find("primary_purpose")) - - try: - if fda_reg_dict[td["nct_id"]] == "false": - td["is_fda_regulated"] = False - elif fda_reg_dict[td["nct_id"]] == "true": - td["is_fda_regulated"] = True - else: - td["is_fda_regulated"] = None - except KeyError: - td["is_fda_regulated"] = None - td["study_status"] = t(soup.overall_status) - - td["start_date"] = (str_to_date(soup.start_date))[0] - - primary_completion_date, td["defaulted_pcd_flag"] = str_to_date( - soup.primary_completion_date - ) - - completion_date, td["defaulted_cd_flag"] = str_to_date( - soup.completion_date - ) - - if not primary_completion_date and not completion_date: - td["available_completion_date"] = None - elif completion_date and not primary_completion_date: - td["available_completion_date"] = completion_date - td["used_primary_completion_date"] = False - else: - td["available_completion_date"] = primary_completion_date - td["used_primary_completion_date"] = True - - if ( - is_interventional(td["study_type"]) - and is_fda_reg(td["fda_reg_drug"], td["fda_reg_device"]) - and is_covered_phase(td["phase"]) - and is_not_device_feasibility(td["primary_purpose"]) - and td["start_date"] - and td["start_date"] >= effective_date - and is_not_withdrawn(td["study_status"]) - ): - td["act_flag"] = True - else: - td["act_flag"] = False - - intervention_type_field = soup.find_all("intervention_type") - trial_intervention_types = [] - for tag in intervention_type_field: - trial_intervention_types.append(tag.get_text()) - - locs = t(soup.location_countries) - - if ( - is_interventional(td["study_type"]) - and is_covered_intervention(trial_intervention_types) - and is_covered_phase(td["phase"]) - and is_not_device_feasibility(td["primary_purpose"]) - and td["available_completion_date"] - and td["available_completion_date"] >= effective_date - and td["start_date"] - and td["start_date"] < effective_date - and is_not_withdrawn(td["study_status"]) - and ( - is_fda_reg(td["fda_reg_drug"], td["fda_reg_device"]) - or is_old_fda_regulated( - td["is_fda_regulated"], - td["fda_reg_drug"], - td["fda_reg_device"], - ) - ) - and has_us_loc(locs) - ): - old_pact_flag = True - else: - old_pact_flag = False - - if ( - is_interventional(td["study_type"]) - and is_fda_reg(td["fda_reg_drug"], td["fda_reg_device"]) - and is_covered_phase(td["phase"]) - and is_not_device_feasibility(td["primary_purpose"]) - and td["start_date"] - and td["start_date"] < effective_date - and td["available_completion_date"] - and td["available_completion_date"] >= effective_date - and is_not_withdrawn(td["study_status"]) - ): - new_pact_flag = True - else: - new_pact_flag = False - - if old_pact_flag == True or new_pact_flag == True: - td["included_pact_flag"] = True - else: - td["included_pact_flag"] = False - - td["location"] = dict_or_none(parsed_json, [cs, "location_countries"]) - - td["has_results"] = does_it_exist(soup.results_first_submitted) - - td["pending_results"] = does_it_exist(soup.pending_results) - - td["pending_data"] = dict_or_none(parsed_json, [cs, "pending_results"]) - - if ( - (td["act_flag"] == True or td["included_pact_flag"] == True) - and date.today() - > td["available_completion_date"] - + relativedelta(years=1) - + timedelta(days=30) - and ( - td["has_certificate"] == 0 - or ( - date.today() - > td["available_completion_date"] - + relativedelta(years=3) - + timedelta(days=30) - ) - ) - ): - td["results_due"] = True - else: - td["results_due"] = False - - td["results_submitted_date"] = ( - str_to_date(soup.results_first_submitted) - )[0] - - td["last_updated_date"] = (str_to_date(soup.last_update_submitted))[0] - - td["certificate_date"] = ( - str_to_date(soup.disposition_first_submitted) - )[0] - - td["enrollment"] = t(soup.enrollment) - if soup.sponsors and soup.sponsors.lead_sponsor: - td["sponsor"] = t(soup.sponsors.lead_sponsor.agency) - td["sponsor_type"] = t(soup.sponsors.lead_sponsor.agency_class) - else: - td["sponsor"] = td["sponsor_type"] = None - - td["collaborators"] = dict_or_none( - parsed_json, [cs, "sponsors", "collaborator"] - ) - - td["exported"] = t(soup.oversight_info and soup.oversight_info.is_us_export) - - td["url"] = t(soup.url) - - td["official_title"] = t(soup.official_title) - - td["brief_title"] = t(soup.brief_title) - - td["title"] = td["official_title"] or td["brief_title"] - - if td["official_title"] is not None: - td["title"] = td["official_title"] - elif td["official_title"] is None and td["brief_title"] is not None: - td["title"] = td["brief_title"] - else: - td["title"] = None - - not_ongoing = [ - "Unknown status", - "Active, not recruiting", - "Not yet recruiting", - "Enrolling by invitation", - "Suspended", - "Recruiting", - ] - if ( - ( - primary_completion_date is None or - primary_completion_date < date.today() - ) - and completion_date is not None - and completion_date < date.today() - and td["study_status"] in not_ongoing - ): - td["discrep_date_status"] = True - else: - td["discrep_date_status"] = False - - if td["certificate_date"] is not None and td["available_completion_date"] is not None: - if td["certificate_date"] > ( - td["available_completion_date"] + relativedelta(years=1) - ): - td["late_cert"] = True - else: - td["late_cert"] = False - else: - td["late_cert"] = False - - if ( - ( - td.get("used_primary_completion_date", False) - and td.get("defaulted_pcd_flag", False) - ) - or - ( - td.get("used_primary_completion_date", False) - and td.get("defaulted_cd_flag", False) - ) - ): - td["defaulted_date"] = True - else: - td["defaulted_date"] = False - - td["condition"] = dict_or_none(parsed_json, [cs, "condition"]) - - td["condition_mesh"] = dict_or_none( - parsed_json, [cs, "condition_browse"] - ) - - td["intervention"] = dict_or_none(parsed_json, [cs, "intervention"]) - - td["intervention_mesh"] = dict_or_none( - parsed_json, [cs, "intervention_browse"] - ) - - td["keywords"] = dict_or_none(parsed_json, [cs, "keyword"]) - if td["act_flag"] or td["included_pact_flag"]: - writer.writerow(convert_bools_to_ints(td)) + cmd = ProcessCommand() + cmd.handle( + input_csv=('https://storage.googleapis.com/ebmdatalab/clinicaltrials/' + 'clinical_trials.csv.tmp')) class Command(BaseCommand): - help = '''Generate a CSV that can be consumed by the `process_data` command, and run that command - ''' + help = ''' Generate a CSV that can be consumed by the `process_data` command, + and run that command ''' + def handle(self, *args, **options): - with contextlib.suppress(OSError): - os.remove(settings.INTERMEDIATE_CSV_PATH) try: - download_and_extract() - convert_to_json() - upload_to_cloud() - convert_to_csv() + convert_data() process_data() + notify_slack("Successful FDAAA import") except: notify_slack("Error in FDAAA import: {}".format(traceback.format_exc())) raise diff --git a/requirements.in b/requirements.in index 1b428ed..8d7b209 100644 --- a/requirements.in +++ b/requirements.in @@ -24,4 +24,5 @@ python-twitter ipython django-extensions ipdb -bs4 \ No newline at end of file +bs4 +-e git://github.com/ebmdatalab/clinicaltrials-act-converter#egg=clinicaltrials_converter \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3a2b14c..6fd5a3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,81 +4,83 @@ # # pip-compile --output-file requirements.txt requirements.in # +-e git+git://github.com/ebmdatalab/clinicaltrials-act-converter#egg=clinicaltrials_converter asn1crypto==0.24.0 # via cryptography backcall==0.1.0 # via ipython bcrypt==3.1.6 # via paramiko beautifulsoup4==4.7.1 # via bs4 bs4==0.0.1 cachetools==3.1.0 # via google-auth -certifi==2018.11.29 # via requests -cffi==1.11.5 # via bcrypt, cryptography, pynacl +certifi==2019.3.9 # via requests +cffi==1.12.3 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -coverage==4.5.2 -coveralls==1.5.1 -cryptography==2.5 # via paramiko +coverage==4.5.3 +coveralls==1.7.0 +cryptography==2.6.1 # via paramiko csscompressor==0.9.5 dateparser==0.7.1 -decorator==4.3.2 # via ipython, traitlets -django-appconf==1.0.2 # via django-compressor +decorator==4.4.0 # via ipython, traitlets +django-appconf==1.0.3 # via django-compressor django-compressor==2.2 -django-extensions==2.1.5 +django-extensions==2.1.6 django-filter==2.1.0 django-libsass==0.7 -django==2.1.7 +django==2.2.1 djangorestframework-csv==2.1.0 -djangorestframework==3.9.1 +djangorestframework==3.9.4 docopt==0.6.2 # via coveralls fabric3==1.14.post1 future==0.17.1 # via python-twitter google-api-python-client==1.6.2 -google-auth==1.6.2 # via google-cloud-bigquery, google-cloud-core, google-cloud-storage +google-auth==1.6.3 # via google-cloud-bigquery, google-cloud-core, google-cloud-storage google-cloud-bigquery==0.26.0 google-cloud-core==0.26.0 # via google-cloud-bigquery, google-cloud-storage google-cloud-storage==1.3.2 google-resumable-media==0.3.2 # via google-cloud-bigquery, google-cloud-storage -googleapis-common-protos==1.5.8 # via google-cloud-core +googleapis-common-protos==1.5.10 # via google-cloud-core gunicorn==19.9.0 -httplib2==0.12.0 # via google-api-python-client, oauth2client +httplib2==0.12.3 # via google-api-python-client, oauth2client idna==2.8 # via requests -ipdb==0.11 +ipdb==0.12 ipython-genutils==0.2.0 # via traitlets -ipython==7.2.0 -jedi==0.13.2 # via ipython -libsass==0.17.0 # via django-libsass -lxml==4.3.1 +ipython==7.5.0 +jedi==0.13.3 # via ipython +libsass==0.18.0 # via django-libsass +lxml==4.3.3 mistune==0.8.4 oauth2client==4.1.3 # via google-api-python-client oauthlib==3.0.1 # via requests-oauthlib paramiko==2.4.2 # via fabric3 -parso==0.3.4 # via jedi -pexpect==4.6.0 # via ipython +parso==0.4.0 # via jedi +pexpect==4.7.0 # via ipython pickleshare==0.7.5 # via ipython -prompt-toolkit==2.0.8 # via ipython -protobuf==3.6.1 # via google-cloud-core, googleapis-common-protos -psycopg2-binary==2.7.7 +prompt-toolkit==2.0.9 # via ipython +protobuf==3.7.1 # via google-cloud-core, googleapis-common-protos +psycopg2-binary==2.8.2 ptyprocess==0.6.0 # via pexpect -pyasn1-modules==0.2.4 # via google-auth, oauth2client +pyasn1-modules==0.2.5 # via google-auth, oauth2client pyasn1==0.4.5 # via oauth2client, paramiko, pyasn1-modules, rsa pycparser==2.19 # via cffi -pygments==2.3.1 # via ipython +pygments==2.4.0 # via ipython pynacl==1.3.0 # via paramiko python-dateutil==2.8.0 # via dateparser python-twitter==3.5 -pytz==2018.9 # via dateparser, django, tzlocal +pytz==2019.1 # via dateparser, django, tzlocal rcssmin==1.0.6 # via django-compressor -regex==2019.2.7 # via dateparser +regex==2019.4.14 # via dateparser requests-oauthlib==1.2.0 # via python-twitter requests==2.21.0 rjsmin==1.0.12 # via django-compressor rsa==4.0 # via google-auth, oauth2client -six==1.12.0 # via bcrypt, cryptography, django-extensions, djangorestframework-csv, fabric3, google-api-python-client, google-auth, google-cloud-core, google-resumable-media, libsass, oauth2client, prompt-toolkit, protobuf, pynacl, python-dateutil, tenacity, traitlets +six==1.12.0 # via bcrypt, cryptography, django-appconf, django-extensions, djangorestframework-csv, fabric3, google-api-python-client, google-auth, google-cloud-core, google-resumable-media, libsass, oauth2client, prompt-toolkit, protobuf, pynacl, python-dateutil, tenacity, traitlets soupsieve==1.9.1 # via beautifulsoup4 +sqlparse==0.3.0 # via django tenacity==4.12.0 # via google-cloud-core traitlets==4.3.2 # via ipython tzlocal==1.5.1 # via dateparser unicodecsv==0.14.1 # via djangorestframework-csv uritemplate==3.0.0 # via google-api-python-client -urllib3==1.24.1 # via requests +urllib3==1.24.3 # via requests wcwidth==0.1.7 # via prompt-toolkit -werkzeug==0.14.1 +werkzeug==0.15.2 xmltodict==0.12.0