From 20e7645cd43167a7b36703c8a47cf904218937aa Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Fri, 30 Nov 2018 14:54:06 -0500 Subject: [PATCH 001/174] does adding the codecov config file make it work? --- codecov.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..e7bb8d289 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,26 @@ +codecov: + notify: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "70...100" + + status: + project: yes + patch: yes + changes: no + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "header, diff" + behavior: default + require_changes: no \ No newline at end of file From 6a3fbc8804ec52f9be4d00f7f9df0c312004be7f Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Fri, 30 Nov 2018 15:03:17 -0500 Subject: [PATCH 002/174] test commit --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index e7bb8d289..2b5037ed5 100644 --- a/codecov.yml +++ b/codecov.yml @@ -23,4 +23,4 @@ parsers: comment: layout: "header, diff" behavior: default - require_changes: no \ No newline at end of file + require_changes: no From 33fe798a357d278f52f8301ae0ffa08c78902ae4 Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Fri, 30 Nov 2018 15:17:45 -0500 Subject: [PATCH 003/174] test 2 --- Jenkinsfile | 2 +- environment.yml | 1 + requirements.txt | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1f9443b97..fbafa8e90 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ bc0.nodetype = "linux-stable" bc0.name = "debug" bc0.build_cmds = ["conda env update --file=environment.yml", "with_env -n jwql python setup.py install"] -bc0.test_cmds = ["with_env -n jwql pytest -s --junitxml=result.xml"] +bc0.test_cmds = ["with_env -n jwql pytest -s --junitxml=result.xml --cov=./ --cov-report xml"] bc0.failedUnstableThresh = 1 bc0.failedFailureThresh = 1 diff --git a/environment.yml b/environment.yml index 7841b2669..b6e14ee8f 100644 --- a/environment.yml +++ b/environment.yml @@ -19,6 +19,7 @@ dependencies: - python=3.6.4 - python-dateutil=2.6.1 - pytest=3.4.2 +- pytest-cov=2.6.0 - sphinx=1.8.1 - sphinx_rtd_theme=0.1.9 - sqlalchemy=1.2.0 diff --git a/requirements.txt b/requirements.txt index 38a114b60..3f9d49e4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ sphinx==1.8.2 sphinx-automodapi==0.9 sqlalchemy==1.2.14 stsci_rtd_theme==0.0.2 +pytest-cov=2.6.0 \ No newline at end of file From cdee457dde17753eb11f557c46929c832779cc82 Mon Sep 17 00:00:00 2001 From: cracraft Date: Mon, 5 Nov 2018 11:03:42 -0500 Subject: [PATCH 004/174] working on adding db for filesystem --- jwql/database/database_interface.py | 65 ++++++++++++++++++++++++ jwql/jwql_monitors/monitor_filesystem.py | 24 +++++---- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 7739beeac..a602df312 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -188,6 +188,71 @@ def colnames(self): return a_list +Class Filesystem(): + """ORM for the filesystem monitor table""" + # Name the table + __tablename__ = 'filesystem' + + # Define the columns + id = Column(Integer, primary_key=True, nullable=False) + date = Column(DateTime, nullable=False, default=datetime.now()) + file_count = Column(Integer, nullable=False) # all files, not just fits + total_size = Column(Float, nullable=False) + used_size = Column(Float, nullable=False) + available_size = Column(Float, nullable=False) + fits_files = Column(Integer, nullable=False) + size_fits = Column(Float, nullable=False) + nrc_uncal_count = Column(Integer, nullable=False) + nrc_cal_count = Column(Integer, nullable=False) + nrc_rate_count = Column(Integer, nullable=False) + nrc_rateints_count = Column(Integer, nullable=False) + nrc_i2d_count = Column(Integer, nullable=False) + nrs_uncal_count = Column(Integer, nullable=False) + nrs_cal_count = Column(Integer, nullable=False) + nrs_rate_count = Column(Integer, nullable=False) + nrs_rateints_count = Column(Integer, nullable=False) + nrs_i2d_count = Column(Integer, nullable=False) + nis_uncal_count = Column(Integer, nullable=False) + nis_cal_count = Column(Integer, nullable=False) + nis_rate_count = Column(Integer, nullable=False) + nis_rateints_count = Column(Integer, nullable=False) + nis_i2d_count = Column(Integer, nullable=False) + mir_uncal_count = Column(Integer, nullable=False) + mir_cal_count = Column(Integer, nullable=False) + mir_rate_count = Column(Integer, nullable=False) + mir_rateints_count = Column(Integer, nullable=False) + mir_i2d_count = Column(Integer, nullable=False) + gui_uncal_count = Column(Integer, nullable=False) + gui_cal_count = Column(Integer, nullable=False) + gui_rate_count = Column(Integer, nullable=False) + gui_rateints_count = Column(Integer, nullable=False) + gui_i2d_count = Column(Integer, nullable=False) + nrc_uncal_size = Column(Float, nullable=False) + nrc_cal_size = Column(Float, nullable=False) + nrc_rate_size = Column(Float, nullable=False) + nrc_rateints_size = Column(Float, nullable=False) + nrc_i2d_size = Column(Float, nullable=False) + nrs_uncal_size = Column(Float, nullable=False) + nrs_cal_size = Column(Float, nullable=False) + nrs_rate_size = Column(Float, nullable=False) + nrs_rateints_size = Column(Float, nullable=False) + nrs_i2d_size = Column(Float, nullable=False) + nis_uncal_size = Column(Float, nullable=False) + nis_cal_size = Column(Float, nullable=False) + nis_rate_size = Column(Float, nullable=False) + nis_rateints_size = Column(Float, nullable=False) + nis_i2d_size = Column(Float, nullable=False) + mir_uncal_size = Column(Float, nullable=False) + mir_cal_size = Column(Float, nullable=False) + mir_rate_size = Column(Float, nullable=False) + mir_rateints_size = Column(Float, nullable=False) + mir_i2d_size = Column(Float, nullable=False) + gui_uncal_size = Column(Float, nullable=False) + gui_cal_size = Column(Float, nullable=False) + gui_rate_size = Column(Float, nullable=False) + gui_rateints_size = Column(Float, nullable=False) + gui_i2d_size = Column(Float, nullable=False) + if __name__ == '__main__': base.metadata.create_all(engine) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index da9890518..dbf1fc495 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -71,6 +71,7 @@ from bokeh.layouts import gridplot from bokeh.plotting import figure, output_file, save +from jwql.database import database_interface as di from jwql.utils.logging_functions import configure_logging, log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.utils import filename_parser @@ -95,6 +96,7 @@ def monitor_filesystem(): # set up dictionaries for output results_dict = defaultdict(int) size_dict = defaultdict(float) + # Walk through all directories recursively and count files logging.info('Searching filesystem...') for dirpath, dirs, files in os.walk(filesystem): @@ -102,15 +104,16 @@ def monitor_filesystem(): for filename in files: file_path = os.path.join(dirpath, filename) if filename.endswith(".fits"): # find total number of fits files - results_dict['fits_files'] += 1 - size_dict['size_fits'] += os.path.getsize(file_path) - suffix = filename_parser(filename)['suffix'] - results_dict[suffix] += 1 - size_dict[suffix] += os.path.getsize(file_path) + suffix = filename_parser(filename)['suffix'] # define suffix detector = filename_parser(filename)['detector'] instrument = detector[0:3] # first three characters of detector specify instrument - results_dict[instrument] += 1 - size_dict[instrument] += os.path.getsize(file_path) + key_count = instrument+'_'+suffix+'_count' + key_size = instrument+'_'+suffix+'_size' + results_dict['fits_files'] += 1 + size_dict['size_fits'] += os.path.getsize(file_path) + results_dict[key_count] += 1 + size_dict[key_size] += os.path.getsize(file_path) + #size_dict[instrument] += os.path.getsize(file_path) logging.info('{} files found in filesystem'.format(results_dict['fits_files'])) # Get df style stats on file system @@ -118,15 +121,18 @@ def monitor_filesystem(): outstring = out.decode("utf-8") # put into string for parsing from byte format parsed = outstring.split(sep=None) - # Select desired elements from parsed string + # Select desired elements from parsed string - put these in dictionary, too? total = int(parsed[8]) # in blocks of 512 bytes used = int(parsed[9]) available = int(parsed[10]) percent_used = parsed[11] - # Save stats for plotting over time + # Save stats for plotting over time # also define this as part of database now = datetime.datetime.now().isoformat(sep='T', timespec='auto') # get date of stats + # change sections below to write to database rather than files + # write in date (now), total, used, available, then the counts and sizes in dictionaries + # set up output file and write stats statsfile = os.path.join(outputs_dir, 'statsfile.txt') with open(statsfile, "a+") as f: From 1e1adb4b4a15e079fa58fcb4cfb0a9f818afa876 Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Mon, 5 Nov 2018 17:05:54 -0500 Subject: [PATCH 005/174] initial population of filesystem stat database tables --- jwql/database/database_interface.py | 28 +++++++++++++++++----- jwql/jwql_monitors/monitor_filesystem.py | 30 +++++++++++++++++++++--- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index a602df312..a97631129 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -188,20 +188,27 @@ def colnames(self): return a_list -Class Filesystem(): - """ORM for the filesystem monitor table""" +class Filesystem_general(): + """ORM for the general (non instrument specific) filesystem monitor table""" # Name the table - __tablename__ = 'filesystem' - + __tablename__ = 'filesystem_general' # Define the columns - id = Column(Integer, primary_key=True, nullable=False) - date = Column(DateTime, nullable=False, default=datetime.now()) + date = Column(DateTime, nullable=False) file_count = Column(Integer, nullable=False) # all files, not just fits total_size = Column(Float, nullable=False) used_size = Column(Float, nullable=False) available_size = Column(Float, nullable=False) fits_files = Column(Integer, nullable=False) size_fits = Column(Float, nullable=False) + + +class Filesystem_instrument(): + """ORM for the instrument specific filesystem monitor table""" + # Name the table + __tablename__ = 'filesystem_instrument' + + # Define the columns + date = Column(DateTime, nullable=False) nrc_uncal_count = Column(Integer, nullable=False) nrc_cal_count = Column(Integer, nullable=False) nrc_rate_count = Column(Integer, nullable=False) @@ -253,6 +260,15 @@ def colnames(self): gui_rateints_size = Column(Float, nullable=False) gui_i2d_size = Column(Float, nullable=False) + @property + def inscolnames(self): + """A list of all the column names in this table EXCEPT the date column""" + # Get the columns + a_list = [col for col, val in self.__dict__.items() + if not isinstance(val, datetime)] + + return a_list + if __name__ == '__main__': base.metadata.create_all(engine) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index dbf1fc495..a62a717b4 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -122,9 +122,9 @@ def monitor_filesystem(): parsed = outstring.split(sep=None) # Select desired elements from parsed string - put these in dictionary, too? - total = int(parsed[8]) # in blocks of 512 bytes - used = int(parsed[9]) - available = int(parsed[10]) + total_size = int(parsed[8]) # in blocks of 512 bytes + used_size = int(parsed[9]) + available_size = int(parsed[10]) percent_used = parsed[11] # Save stats for plotting over time # also define this as part of database @@ -133,6 +133,29 @@ def monitor_filesystem(): # change sections below to write to database rather than files # write in date (now), total, used, available, then the counts and sizes in dictionaries + + current_columns = di.Filesystem_instrument.inscolnames + + # add row to filesystem_general table + di.session.add(di.Filesystem_general(date=now, total_size=total_size, + used_size=used_size, + available_size=available_size, + file_count = results_dict['file_count'], + fits_files = results_dict['fits_files'], + size_fits = size_dict['size_fits'])) + + # Add row to filesystem_instrument table + combo_dict = {**results_dict, **size_dict} #fancy dictionary stuff + filtered_dict = {k: combo_dict.get(k, 0) for k in current_columns} # more fancy dict stuff + di.session.add(di.Filesystem_instrument(date=now, **filtered_dict)) + + # Commit new rows to database + di.session.commit() + + + # Plotting goes here! + + ''' # set up output file and write stats statsfile = os.path.join(outputs_dir, 'statsfile.txt') with open(statsfile, "a+") as f: @@ -165,6 +188,7 @@ def monitor_filesystem(): # Create the plots plot_system_stats(statsfile, filesbytype, sizebytype) + ''' def plot_system_stats(stats_file, filebytype, sizebytype): From 105364d98e1fb753162e1d9f84ac8427e24ae5b3 Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Tue, 20 Nov 2018 10:50:23 -0500 Subject: [PATCH 006/174] missed base superclass --- jwql/database/database_interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index a97631129..9ea95a2d3 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -188,7 +188,7 @@ def colnames(self): return a_list -class Filesystem_general(): +class Filesystem_general(base): """ORM for the general (non instrument specific) filesystem monitor table""" # Name the table __tablename__ = 'filesystem_general' @@ -202,7 +202,7 @@ class Filesystem_general(): size_fits = Column(Float, nullable=False) -class Filesystem_instrument(): +class Filesystem_instrument(base): """ORM for the instrument specific filesystem monitor table""" # Name the table __tablename__ = 'filesystem_instrument' From c2108785f7ee931e08efc7f2a41a30f61ac8f84b Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Mon, 26 Nov 2018 13:47:47 -0500 Subject: [PATCH 007/174] squash --- jwql/jwql_monitors/monitor_filesystem.py | 112 ++++++++++++++--------- 1 file changed, 67 insertions(+), 45 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index a62a717b4..02318dbb4 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -67,6 +67,8 @@ import os import subprocess +from sqlalchemy import func + from bokeh.embed import components from bokeh.layouts import gridplot from bokeh.plotting import figure, output_file, save @@ -125,7 +127,6 @@ def monitor_filesystem(): total_size = int(parsed[8]) # in blocks of 512 bytes used_size = int(parsed[9]) available_size = int(parsed[10]) - percent_used = parsed[11] # Save stats for plotting over time # also define this as part of database now = datetime.datetime.now().isoformat(sep='T', timespec='auto') # get date of stats @@ -133,9 +134,6 @@ def monitor_filesystem(): # change sections below to write to database rather than files # write in date (now), total, used, available, then the counts and sizes in dictionaries - - current_columns = di.Filesystem_instrument.inscolnames - # add row to filesystem_general table di.session.add(di.Filesystem_general(date=now, total_size=total_size, used_size=used_size, @@ -145,6 +143,9 @@ def monitor_filesystem(): size_fits = size_dict['size_fits'])) # Add row to filesystem_instrument table + current_columns = [t.name for t in di.Filesystem_instrument.metadata.sorted_tables[2].columns] + current_columns.remove('date') + combo_dict = {**results_dict, **size_dict} #fancy dictionary stuff filtered_dict = {k: combo_dict.get(k, 0) for k in current_columns} # more fancy dict stuff di.session.add(di.Filesystem_instrument(date=now, **filtered_dict)) @@ -155,40 +156,40 @@ def monitor_filesystem(): # Plotting goes here! - ''' + # set up output file and write stats - statsfile = os.path.join(outputs_dir, 'statsfile.txt') - with open(statsfile, "a+") as f: - f.write("{0} {1:15d} {2:15d} {3:15d} {4:15d} {5}\n".format(now, results_dict['file_count'], - total, available, used, percent_used)) - set_permissions(statsfile) - logging.info('Saved file statistics to: {}'.format(statsfile)) - - # set up and read out stats on files by type - filesbytype = os.path.join(outputs_dir, 'filesbytype.txt') - with open(filesbytype, "a+") as f2: - f2.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(results_dict['fits_files'], - results_dict['uncal'], results_dict['cal'], results_dict['rate'], - results_dict['rateints'], results_dict['i2d'], results_dict['nrc'], - results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui'])) - set_permissions(filesbytype, verbose=False) - logging.info('Saved file statistics by type to {}'.format(filesbytype)) - - # set up file size by type file - sizebytype = os.path.join(outputs_dir, 'sizebytype.txt') - with open(sizebytype, "a+") as f3: - f3.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(size_dict['size_fits'], - size_dict['uncal'], size_dict['cal'], size_dict['rate'], - size_dict['rateints'], size_dict['i2d'], size_dict['nrc'], - size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui'])) - set_permissions(sizebytype, verbose=False) - logging.info('Saved file sizes by type to {}'.format(sizebytype)) - - logging.info('Filesystem statistics calculation complete.') + # statsfile = os.path.join(outputs_dir, 'statsfile.txt') + # with open(statsfile, "a+") as f: + # f.write("{0} {1:15d} {2:15d} {3:15d} {4:15d} {5}\n".format(now, results_dict['file_count'], + # total, available, used, percent_used)) + # set_permissions(statsfile) + # logging.info('Saved file statistics to: {}'.format(statsfile)) + # + # # set up and read out stats on files by type + # filesbytype = os.path.join(outputs_dir, 'filesbytype.txt') + # with open(filesbytype, "a+") as f2: + # f2.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(results_dict['fits_files'], + # results_dict['uncal'], results_dict['cal'], results_dict['rate'], + # results_dict['rateints'], results_dict['i2d'], results_dict['nrc'], + # results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui'])) + # set_permissions(filesbytype, verbose=False) + # logging.info('Saved file statistics by type to {}'.format(filesbytype)) + # + # # set up file size by type file + # sizebytype = os.path.join(outputs_dir, 'sizebytype.txt') + # with open(sizebytype, "a+") as f3: + # f3.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(size_dict['size_fits'], + # size_dict['uncal'], size_dict['cal'], size_dict['rate'], + # size_dict['rateints'], size_dict['i2d'], size_dict['nrc'], + # size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui'])) + # set_permissions(sizebytype, verbose=False) + # logging.info('Saved file sizes by type to {}'.format(sizebytype)) + # + # logging.info('Filesystem statistics calculation complete.') # Create the plots plot_system_stats(statsfile, filesbytype, sizebytype) - ''' + def plot_system_stats(stats_file, filebytype, sizebytype): @@ -205,22 +206,43 @@ def plot_system_stats(stats_file, filebytype, sizebytype): file containing information on file sizes by type over time """ + # pull up database tables + filesystem_general = di.session.query(di.Filesystem_general) + filesystem_instrument = di.session.query(di.Filesystem_instrument) + + # get path for files - settings = get_config() - outputs_dir = os.path.join(settings['outputs'], 'monitor_filesystem') + #settings = get_config() # read in file of statistics - date, f_count, sysize, frsize, used, percent = np.loadtxt(stats_file, dtype=str, unpack=True) - fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(filebytype, dtype=str, unpack=True) - fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(sizebytype, dtype=str, unpack=True) + date = filesystem_general.date + file_count = filesystem_general.file_count + sy_size = filesystem_general.total_size + used_size = filesystem_general.used_size + free_size = filesystem_general.available_size + fits_files = filesystem_general.fits_files + fits_sz = filesystem_general.size_fits + + uncal_size = di.session.query(func.count( + di.filesystem_instrument.filter(di.filesystem_instrument.column.like('*_uncal*')))) / (1024.**3) + + + #uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, \ + #nisfiles, mirfiles, fgsfiles = np.loadtxt(filebytype, dtype=str, unpack=True) + + + #uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, \ + #mir_sz, fgs_sz = np.loadtxt(sizebytype, dtype=str, unpack=True) + + logging.info('Read in file statistics from {}, {}, {}'.format(stats_file, filebytype, sizebytype)) # put in proper np array types and convert to GB sizes dates = np.array(date, dtype='datetime64') - file_count = f_count.astype(float) - systemsize = sysize.astype(float) / (1024.**3) - freesize = frsize.astype(float) / (1024.**3) - usedsize = used.astype(float) / (1024.**3) + systemsize = sy_size / (1024.**3) + freesize = free_size / (1024.**3) + usedsize = used_size / (1024.**3) + fits_size = fits_sz/ (1024.**3) fits = fits_files.astype(int) uncal = uncalfiles.astype(int) @@ -234,8 +256,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype): miri = mirfiles.astype(int) fgs = fgsfiles.astype(int) - fits_size = fits_sz.astype(float) / (1024.**3) - uncal_size = uncal_sz.astype(float) / (1024.**3) + + #uncal_size = uncal_sz.astype(float) / (1024.**3) cal_size = cal_sz.astype(float) / (1024.**3) rate_size = rate_sz.astype(float) / (1024.**3) rateints_size = rateints_sz.astype(float) / (1024.**3) From 0406f61f7fe30f9983c84faed62b728c417151cd Mon Sep 17 00:00:00 2001 From: Sara Ogaz Date: Tue, 11 Dec 2018 15:09:40 -0500 Subject: [PATCH 008/174] rework of filesystem database --- jwql/database/database_interface.py | 5 +- jwql/jwql_monitors/monitor_filesystem.py | 356 ++++++++++++++--------- 2 files changed, 229 insertions(+), 132 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 9ea95a2d3..d5fa459f1 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -62,6 +62,7 @@ from sqlalchemy import create_engine from sqlalchemy import DateTime from sqlalchemy import Integer +from sqlalchemy import Float from sqlalchemy import MetaData from sqlalchemy import String from sqlalchemy.ext.declarative import declarative_base @@ -193,7 +194,7 @@ class Filesystem_general(base): # Name the table __tablename__ = 'filesystem_general' # Define the columns - date = Column(DateTime, nullable=False) + date = Column(DateTime, primary_key=True, nullable=False) file_count = Column(Integer, nullable=False) # all files, not just fits total_size = Column(Float, nullable=False) used_size = Column(Float, nullable=False) @@ -208,7 +209,7 @@ class Filesystem_instrument(base): __tablename__ = 'filesystem_instrument' # Define the columns - date = Column(DateTime, nullable=False) + date = Column(DateTime, primary_key=True, nullable=False) nrc_uncal_count = Column(Integer, nullable=False) nrc_cal_count = Column(Integer, nullable=False) nrc_rate_count = Column(Integer, nullable=False) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 02318dbb4..96d1ac432 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -9,7 +9,7 @@ Authors ------- - - Misty Cracraft + - Misty Cracraft, Sara Ogaz Use --- @@ -60,20 +60,19 @@ as saving them to an output html file. """ -from collections import defaultdict import datetime import logging import numpy as np import os import subprocess -from sqlalchemy import func - from bokeh.embed import components from bokeh.layouts import gridplot from bokeh.plotting import figure, output_file, save from jwql.database import database_interface as di +from jwql.database.database_interface import Filesystem_general +from jwql.database.database_interface import Filesystem_instrument as InsDb from jwql.utils.logging_functions import configure_logging, log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.utils import filename_parser @@ -83,21 +82,21 @@ @log_fail @log_info def monitor_filesystem(): - """Tabulates the inventory of the JWST filesystem, saving + """ + Tabulates the inventory of the JWST filesystem, saving statistics to files, and generates plots. """ # Begin logging logging.info('Beginning filesystem monitoring.') - # Get path, directories and files in system and count files in all directories + # Get path, dirs and files in system and count files in all directories settings = get_config() filesystem = settings['filesystem'] - outputs_dir = os.path.join(settings['outputs'], 'monitor_filesystem') # set up dictionaries for output - results_dict = defaultdict(int) - size_dict = defaultdict(float) + results_dict = {} + size_dict = {} # Walk through all directories recursively and count files logging.info('Searching filesystem...') @@ -108,165 +107,260 @@ def monitor_filesystem(): if filename.endswith(".fits"): # find total number of fits files suffix = filename_parser(filename)['suffix'] # define suffix detector = filename_parser(filename)['detector'] - instrument = detector[0:3] # first three characters of detector specify instrument + # first three characters of detector specify instrument + instrument = detector[0:3] key_count = instrument+'_'+suffix+'_count' key_size = instrument+'_'+suffix+'_size' results_dict['fits_files'] += 1 size_dict['size_fits'] += os.path.getsize(file_path) results_dict[key_count] += 1 size_dict[key_size] += os.path.getsize(file_path) - #size_dict[instrument] += os.path.getsize(file_path) - logging.info('{} files found in filesystem'.format(results_dict['fits_files'])) + # size_dict[instrument] += os.path.getsize(file_path) + logging.info('{} files found in filesystem'.format( + results_dict['fits_files'])) # Get df style stats on file system out = subprocess.check_output('df {}'.format(filesystem), shell=True) - outstring = out.decode("utf-8") # put into string for parsing from byte format + # put into string for parsing from byte format + outstring = out.decode("utf-8") parsed = outstring.split(sep=None) - # Select desired elements from parsed string - put these in dictionary, too? + # Select desired elements from parsed string, put these in dictionary, too? total_size = int(parsed[8]) # in blocks of 512 bytes used_size = int(parsed[9]) available_size = int(parsed[10]) # Save stats for plotting over time # also define this as part of database - now = datetime.datetime.now().isoformat(sep='T', timespec='auto') # get date of stats - - # change sections below to write to database rather than files - # write in date (now), total, used, available, then the counts and sizes in dictionaries + # get date of stats + now = datetime.datetime.now().isoformat(sep='T') # add row to filesystem_general table di.session.add(di.Filesystem_general(date=now, total_size=total_size, used_size=used_size, available_size=available_size, - file_count = results_dict['file_count'], - fits_files = results_dict['fits_files'], - size_fits = size_dict['size_fits'])) + file_count=results_dict['file_count'], + fits_files=results_dict['fits_files'], + size_fits=size_dict['size_fits'])) # Add row to filesystem_instrument table - current_columns = [t.name for t in di.Filesystem_instrument.metadata.sorted_tables[2].columns] + current_columns = [t.name for t in + di.Filesystem_instrument.metadata.sorted_tables[2].columns] current_columns.remove('date') - combo_dict = {**results_dict, **size_dict} #fancy dictionary stuff - filtered_dict = {k: combo_dict.get(k, 0) for k in current_columns} # more fancy dict stuff + # fancy dictionary stuff + combo_dict = {**results_dict, **size_dict} + # more fancy dict stuff + filtered_dict = {k: combo_dict.get(k, 0) for k in current_columns} di.session.add(di.Filesystem_instrument(date=now, **filtered_dict)) # Commit new rows to database di.session.commit() - - # Plotting goes here! - - - # set up output file and write stats - # statsfile = os.path.join(outputs_dir, 'statsfile.txt') - # with open(statsfile, "a+") as f: - # f.write("{0} {1:15d} {2:15d} {3:15d} {4:15d} {5}\n".format(now, results_dict['file_count'], - # total, available, used, percent_used)) - # set_permissions(statsfile) - # logging.info('Saved file statistics to: {}'.format(statsfile)) - # - # # set up and read out stats on files by type - # filesbytype = os.path.join(outputs_dir, 'filesbytype.txt') - # with open(filesbytype, "a+") as f2: - # f2.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(results_dict['fits_files'], - # results_dict['uncal'], results_dict['cal'], results_dict['rate'], - # results_dict['rateints'], results_dict['i2d'], results_dict['nrc'], - # results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui'])) - # set_permissions(filesbytype, verbose=False) - # logging.info('Saved file statistics by type to {}'.format(filesbytype)) - # - # # set up file size by type file - # sizebytype = os.path.join(outputs_dir, 'sizebytype.txt') - # with open(sizebytype, "a+") as f3: - # f3.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(size_dict['size_fits'], - # size_dict['uncal'], size_dict['cal'], size_dict['rate'], - # size_dict['rateints'], size_dict['i2d'], size_dict['nrc'], - # size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui'])) - # set_permissions(sizebytype, verbose=False) - # logging.info('Saved file sizes by type to {}'.format(sizebytype)) - # - # logging.info('Filesystem statistics calculation complete.') - # Create the plots - plot_system_stats(statsfile, filesbytype, sizebytype) - - + plot_system_stats() -def plot_system_stats(stats_file, filebytype, sizebytype): - """Read in the file of saved stats over time and plot them. - Parameters - ----------- - stats_file : str - file containing information of stats over time - filebytype : str - file containing information of file counts by type over - time - sizebytype : str - file containing information on file sizes by type over time +def plot_system_stats(): + """ + Read in the file of saved stats over time and plot them. """ - - # pull up database tables - filesystem_general = di.session.query(di.Filesystem_general) - filesystem_instrument = di.session.query(di.Filesystem_instrument) - # get path for files - #settings = get_config() + settings = get_config() + outputs_dir = os.path.join(settings['outputs'], 'monitor_filesystem') # read in file of statistics - date = filesystem_general.date - file_count = filesystem_general.file_count - sy_size = filesystem_general.total_size - used_size = filesystem_general.used_size - free_size = filesystem_general.available_size - fits_files = filesystem_general.fits_files - fits_sz = filesystem_general.size_fits - - uncal_size = di.session.query(func.count( - di.filesystem_instrument.filter(di.filesystem_instrument.column.like('*_uncal*')))) / (1024.**3) - - - #uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, \ - #nisfiles, mirfiles, fgsfiles = np.loadtxt(filebytype, dtype=str, unpack=True) - - - #uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, \ - #mir_sz, fgs_sz = np.loadtxt(sizebytype, dtype=str, unpack=True) - - - logging.info('Read in file statistics from {}, {}, {}'.format(stats_file, filebytype, sizebytype)) + general_query = di.session.query(Filesystem_general.date, + Filesystem_general.file_count, + Filesystem_general.total_size, + Filesystem_general.used_size, + Filesystem_general.available_size, + Filesystem_general.fits_files, + Filesystem_general.size_fits).all() + date_list_general, file_count, sy_size, used_size, free_size, fits_files, \ + fits_sz = [], [], [], [], [], [], [] + + for row in general_query: + date_list_general.append(row[0]) + file_count.append(row[1]) + sy_size.append(row[2]) + used_size.append(row[3]) + free_size.append(row[4]) + fits_files.append(row[5]) + fits_sz.append(row[6]) # put in proper np array types and convert to GB sizes - dates = np.array(date, dtype='datetime64') - systemsize = sy_size / (1024.**3) - freesize = free_size / (1024.**3) - usedsize = used_size / (1024.**3) - fits_size = fits_sz/ (1024.**3) - - fits = fits_files.astype(int) - uncal = uncalfiles.astype(int) - cal = calfiles.astype(int) - rate = ratefiles.astype(int) - rateints = rateintsfiles.astype(int) - i2d = i2dfiles.astype(int) - nircam = nrcfiles.astype(int) - nirspec = nrsfiles.astype(int) - niriss = nisfiles.astype(int) - miri = mirfiles.astype(int) - fgs = fgsfiles.astype(int) - - - #uncal_size = uncal_sz.astype(float) / (1024.**3) - cal_size = cal_sz.astype(float) / (1024.**3) - rate_size = rate_sz.astype(float) / (1024.**3) - rateints_size = rateints_sz.astype(float) / (1024.**3) - i2d_size = i2d_sz.astype(float) / (1024.**3) - nircam_size = nrc_sz.astype(float) / (1024.**3) - nirspec_size = nrs_sz.astype(float) / (1024.**3) - niriss_size = nis_sz.astype(float) / (1024.**3) - miri_size = mir_sz.astype(float) / (1024.**3) - fgs_size = fgs_sz.astype(float) / (1024.**3) + dates = np.array(date_list_general, dtype='datetime64') + systemsize = np.array(sy_size) / (1024.**3) + freesize = np.array(free_size) / (1024.**3) + usedsize = np.array(used_size) / (1024.**3) + fits_size = np.array(fits_sz) / (1024.**3) + fits = fits_files + + # Information from Filesystem_instrument + # There is a better way to do this... but we don't know what the column + # name lists will be ahead of time, so from what I can tell, we have to + # either hard code the column names or do the summation after the query. + uncal_filecount_results = di.session.query((InsDb.nrc_uncal_count + + InsDb.nrs_uncal_count + + InsDb.nis_uncal_count + + InsDb.mir_uncal_count + + InsDb.gui_uncal_count). + label("uncal_count")).all() + uncal = [row[0] for row in uncal_filecount_results] + + cal_filecount_results = di.session.query((InsDb.nrc_cal_count + + InsDb.nrs_cal_count + + InsDb.nis_cal_count + + InsDb.mir_cal_count + + InsDb.gui_cal_count). + label("cal_count")).all() + cal = [row[0] for row in cal_filecount_results] + + rate_filecount_results = di.session.query((InsDb.nrc_rate_count + + InsDb.nrs_rate_count + + InsDb.nis_rate_count + + InsDb.mir_rate_count + + InsDb.gui_rate_count). + label("rate_count")).all() + rate = [row[0] for row in rate_filecount_results] + + rateints_filecount_results = di.session.query((InsDb.nrc_rateints_count + + InsDb.nrs_rateints_count + + InsDb.nis_rateints_count + + InsDb.mir_rateints_count + + InsDb.gui_rateints_count). + label("rateints_count")).all() + rateints = [row[0] for row in rateints_filecount_results] + + i2d_filecount_results = di.session.query((InsDb.nrc_i2d_count + + InsDb.nrs_i2d_count + + InsDb.nis_i2d_count + + InsDb.mir_i2d_count + + InsDb.gui_i2d_count). + label("i2d_count")).all() + i2d = [row[0] for row in i2d_filecount_results] + + nircam_filecount_results = di.session.query((InsDb.nrc_uncal_count + + InsDb.nrc_cal_count + + InsDb.nrc_rate_count + + InsDb.nrc_rateints_count + + InsDb.nrc_i2d_count). + label("nir_count")).all() + nircam = [row[0] for row in nircam_filecount_results] + + nirspec_filecount_results = di.session.query((InsDb.nrs_uncal_count + + InsDb.nrs_cal_count + + InsDb.nrs_rate_count + + InsDb.nrs_rateints_count + + InsDb.nrs_i2d_count). + label("nrs_count")).all() + nirspec = [row[0] for row in nirspec_filecount_results] + + niriss_filecount_results = di.session.query((InsDb.nis_uncal_count + + InsDb.nis_cal_count + + InsDb.nis_rate_count + + InsDb.nis_rateints_count + + InsDb.nis_i2d_count). + label("nis_count")).all() + niriss = [row[0] for row in niriss_filecount_results] + + miri_filecount_results = di.session.query((InsDb.mir_uncal_count + + InsDb.mir_cal_count + + InsDb.mir_rate_count + + InsDb.mir_rateints_count + + InsDb.mir_i2d_count). + label("mir_count")).all() + miri = [row[0] for row in miri_filecount_results] + + fgs_filecount_results = di.session.query((InsDb.gui_uncal_count + + InsDb.gui_cal_count + + InsDb.gui_rate_count + + InsDb.gui_rateints_count + + InsDb.gui_i2d_count). + label("nir_count")).all() + fgs = [row[0] for row in fgs_filecount_results] + + # Next is the size set + uncal_size_results = di.session.query((InsDb.nrc_uncal_size + + InsDb.nrs_uncal_size + + InsDb.nis_uncal_size + + InsDb.mir_uncal_size + + InsDb.gui_uncal_size). + label("uncal_size")).all() + uncal_size = np.array([row[0] for row in uncal_size_results]) / (1024.**3) + + cal_size_results = di.session.query((InsDb.nrc_cal_size + + InsDb.nrs_cal_size + + InsDb.nis_cal_size + + InsDb.mir_cal_size + + InsDb.gui_cal_size). + label("cal_size")).all() + cal_size = np.array([row[0] for row in cal_size_results]) / (1024.**3) + + rate_size_results = di.session.query((InsDb.nrc_rate_size + + InsDb.nrs_rate_size + + InsDb.nis_rate_size + + InsDb.mir_rate_size + + InsDb.gui_rate_size). + label("rate_size")).all() + rate_size = np.array([row[0] for row in rate_size_results]) / (1024.**3) + + rateints_size_results = di.session.query((InsDb.nrc_rateints_size + + InsDb.nrs_rateints_size + + InsDb.nis_rateints_size + + InsDb.mir_rateints_size + + InsDb.gui_rateints_size). + label("rateints_size")).all() + rateints_size = np.array([row[0] for row in rateints_size_results]) / (1024.**3) + + i2d_size_results = di.session.query((InsDb.nrc_i2d_size + + InsDb.nrs_i2d_size + + InsDb.nis_i2d_size + + InsDb.mir_i2d_size + + InsDb.gui_i2d_size). + label("i2d_size")).all() + i2d_size = np.array([row[0] for row in i2d_size_results]) / (1024.**3) + + nircam_size_results = di.session.query((InsDb.nrc_uncal_size + + InsDb.nrc_cal_size + + InsDb.nrc_rate_size + + InsDb.nrc_rateints_size + + InsDb.nrc_i2d_size). + label("nrc_size")).all() + nircam_size = np.array([row[0] for row in nircam_size_results]) / (1024.**3) + + nirspec_size_results = di.session.query((InsDb.nrs_uncal_size + + InsDb.nrs_cal_size + + InsDb.nrs_rate_size + + InsDb.nrs_rateints_size + + InsDb.nrs_i2d_size). + label("nrs_size")).all() + nirspec_size = np.array([row[0] for row in nirspec_size_results]) / (1024.**3) + + niriss_size_results = di.session.query((InsDb.nis_uncal_size + + InsDb.nis_cal_size + + InsDb.nis_rate_size + + InsDb.nis_rateints_size + + InsDb.nis_i2d_size). + label("nis_size")).all() + niriss_size = np.array([row[0] for row in niriss_size_results]) / (1024.**3) + + miri_size_results = di.session.query((InsDb.mir_uncal_size + + InsDb.mir_cal_size + + InsDb.mir_rate_size + + InsDb.mir_rateints_size + + InsDb.mir_i2d_size). + label("mir_size")).all() + miri_size = np.array([row[0] for row in miri_size_results]) / (1024.**3) + + fgs_size_results = di.session.query((InsDb.gui_uncal_size + + InsDb.gui_cal_size + + InsDb.gui_rate_size + + InsDb.gui_rateints_size + + InsDb.gui_i2d_size). + label("mir_size")).all() + fgs_size = np.array([row[0] for row in fgs_size_results]) / (1024.**3) # plot the data # Plot filecount vs. date @@ -296,7 +390,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p3.line(dates, uncal, legend='uncalibrated fits files', line_color='red') p3.diamond(dates, uncal, color='red') p3.line(dates, cal, legend='calibrated fits files', line_color='blue') - p3.square(date, cal, color='blue') + # This was original "date", what did we want here? + p3.square(dates, cal, color='blue') p3.line(dates, rate, legend='rate fits files', line_color='green') p3.triangle(dates, rate, color='green') p3.line(dates, rateints, legend='rateints fits files', line_color='orange') @@ -323,7 +418,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p4.line(dates, uncal_size, legend='uncalibrated fits files', line_color='red') p4.diamond(dates, uncal_size, color='red') p4.line(dates, cal_size, legend='calibrated fits files', line_color='blue') - p4.square(date, cal_size, color='blue') + # This was original "date", what did we want here? + p4.square(dates, cal_size, color='blue') p4.line(dates, rate_size, legend='rate fits files', line_color='green') p4.triangle(dates, rate_size, color='green') p4.line(dates, rateints_size, legend='rateints fits files', line_color='orange') From fa14f7de2360a0da1dabf0fab2f33fce5592143e Mon Sep 17 00:00:00 2001 From: DanielKuebi Date: Thu, 14 Feb 2019 10:57:46 -0500 Subject: [PATCH 009/174] initial commit --- .../data_trending/15min_to_db.py | 104 +++++ .../data_trending/MIRI_mnemonics.db | Bin 0 -> 401408 bytes .../miri_monitors/data_trending/condition.py | 352 ++++++++++++++ .../data_trending/csv_to_AstropyTable.py | 150 ++++++ .../data_trending/data_extract.py | 287 ++++++++++++ .../miri_monitors/data_trending/day_to_db.py | 154 +++++++ .../data_trending/filterpos_extract.py | 112 +++++ .../miri_monitors/data_trending/main.py | 143 ++++++ .../miri_monitors/data_trending/mnemonics.py | 428 ++++++++++++++++++ .../miri_monitors/data_trending/plotting.py | 92 ++++ .../data_trending/sql_interface.py | 186 ++++++++ .../data_trending/test_query_data.py | 79 ++++ .../data_trending/test_query_pos.py | 79 ++++ .../miri_monitors/data_trending/trash.py | 275 +++++++++++ jwql/utils/edb.py | 40 ++ 15 files changed, 2481 insertions(+) create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/15min_to_db.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/MIRI_mnemonics.db create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/condition.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/csv_to_AstropyTable.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/data_extract.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/filterpos_extract.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/main.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/mnemonics.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/plotting.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/sql_interface.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/test_query_data.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/test_query_pos.py create mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/trash.py create mode 100644 jwql/utils/edb.py diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/15min_to_db.py b/jwql/instrument_monitors/miri_monitors/data_trending/15min_to_db.py new file mode 100755 index 000000000..9b40cc98b --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/15min_to_db.py @@ -0,0 +1,104 @@ + +import statistics +import mnemonics as mn +import sql_interface as sql +import csv_to_AstropyTable as apt + +from data_extract import once_a_day_routine + + + +#create filename string +directory = '/home/daniel/STScI/trainigData/set_1_15min_complete/' + +filenames1 = [ +'imir_190130_otis229FOFTLM2019030204146194.CSV', +'imir_190130_otis230FOFTLM2019030204240886.CSV', +'imir_190130_otis241FOFTLM2019030210651672.CSV', +'imir_190130_otis240FOFTLM2019030210631185.CSV', +'imir_190130_otis231FOFTLM2019030204334644.CSV', +'imir_190130_otis233FOFTLM2019030204521412.CSV', +'imir_190130_otis242FOFTLM2019030210728909.CSV', +'imir_190130_otis232FOFTLM2019030204455835.CSV', +'imir_190130_otis243FOFTLM2019030210744062.CSV', +'imir_190130_otis233FOFTLM2019030204521412.CSV', +'imir_190130_otis244FOFTLM2019030210809362.CSV', +'imir_190130_otis234FOFTLM2019030204555665.CSV', +'imir_190130_otis245FOFTLM2019030210828095.CSV', +'imir_190130_otis235FOFTLM2019030204617145.CSV', +'imir_190130_otis246FOFTLM2019030210852965.CSV', +'imir_190130_otis236FOFTLM2019030204651604.CSV', +'imir_190130_otis247FOFTLM2019030210914141.CSV', +'imir_190130_otis237FOFTLM2019030204712019.CSV', +'imir_190130_otis248FOFTLM2019030210940944.CSV', +'imir_190130_otis238FOFTLM2019030204738855.CSV', +'imir_190130_otis249FOFTLM2019030211002524.CSV', +'imir_190130_otis239FOFTLM2019030204805611.CSV'] + +filenames = [ +'imir_190130_otis229FOFTLM2019030204146194.CSV', +'imir_190130_otis234FOFTLM2019030204555665.CSV', +'imir_190130_otis230FOFTLM2019030204240886.CSV', +'imir_190130_otis235FOFTLM2019030204617145.CSV', +'imir_190130_otis231FOFTLM2019030204334644.CSV', +'imir_190130_otis236FOFTLM2019030204651604.CSV', +'imir_190130_otis232FOFTLM2019030204455835.CSV', +'imir_190130_otis237FOFTLM2019030204712019.CSV', +'imir_190130_otis233FOFTLM2019030204521412.CSV', +'imir_190130_otis238FOFTLM2019030204738855.CSV'] + +def process_file(conn, path): + m_raw_data = apt.mnemonics(path) + + cond1, cond2 = once_a_day_routine(m_raw_data) + + for key, value in cond1.items(): + + m = m_raw_data.mnemonic(key) + + if key == "SE_ZIMIRICEA": + length = len(value) + mean = statistics.mean(value) + deviation = statistics.stdev(value) + + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, "SE_ZIMIRICEA_IDLE", dataset) + + elif key == "IMIR_HK_ICE_SEC_VOLT4": + length = len(value) + mean = statistics.mean(value) + deviation = statistics.stdev(value) + + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, "IMIR_HK_ICE_SEC_VOLT4_IDLE", dataset) + + else: + length = len(value) + mean = statistics.mean(value) + deviation = statistics.stdev(value) + + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, key, dataset) + + for key, value in cond2.items(): + length = len(value) + mean = statistics.mean(value) + deviation = statistics.stdev(value) + + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, key, dataset) + + +def main(): + db_file = "miri_database.db" + conn = sql.create_connection(db_file) + + for name in filenames: + path = directory + name + process_file(conn, path) + + sql.close_connection(conn) + print("done") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/MIRI_mnemonics.db b/jwql/instrument_monitors/miri_monitors/data_trending/MIRI_mnemonics.db new file mode 100755 index 0000000000000000000000000000000000000000..162be971209e71b5b2d868981b7b76ea10470855 GIT binary patch literal 401408 zcmeEv2Urxz);2+cWK0A>QHcfwhV0-Oa)u!)B9cW6fQXm_k`Z&xV$Pzj*;UM#5ffro zz=Vo9>t79{FkMh{?_KWq?A^cSd6vbVemPZdpYEBr&Z%<({Ji2*Vs!>4CJ&2D(cy@x zh>44f@pN>=#3a9oiHSv_|9-vv(Jw82{R919yut5H{EGgDvCbkX^sbWf0p^?t<*&*g zlwT`XD?e6#sC-AcO!k38K4YM1}FoR0m=Yn zfHLs^F9RXoXzl^-=!drx`r!iT2iTw=)*ST1tOxqh%@F<2?TmhCYoQ-%?a&WZRrEtn z0sWAbRzyGB$g7DdA61^B?5NyA>7-J&lABUX#j}c26}=P{6)q~wP~a=5%3qP6D<6bT zq%X<k38K4YM1}FoR0m{HXj{$BQSuq(IaTys7ubR)7>U6V;`G1gpk`Es)YHg7< z)vtv4$78lS<_UfZyOH&2r&j5%1<#82#WKZH zFgVBDPN5#oFBDiWtc4#&#K*3+JDvd^(Qd`A(a9Lx%-l}C9`09D2luO~h2MY+7rrm= zfA2GRE`8WZ>)~}4+|t&<#!jvt&Yvl;KC>47CtTRHFLz){7-D|)r=;LD7~I^(u4O&k zuS8(Iq!zv&Szmak*TyvPoZ?Vou}l?%b8PHn>*4$)f%T+X_yc6UYuckJJHYKy*^j~Z z3NW~tjh#$A+^3i#1l;GgK9#M#K^SgkV{2-`m1f|-?l*yTJ_8Se zOLOzQZd-@L2J6dDYgRjBXcJ32sRqUc%=yi%HC}z%*QZELt1*cs+4T3SanU$Sn12|tW_1dwEZG#KDBMRQVItU)c zx8?Q?jlIo-+dH`3DJaq5)KxSoM5IJ#p@?ZoThz9n^V-;!E* z8G;WVDCHXgo}I?~&4}-g!Oc00{zpN7`l1X_1}FoR0m=YnfHFWCpbSt3Cd`vk**;!dysaPpb$xBH|u|#o}Vt`^>g&PWs6halW5DQz8K4YM z1}FoR0m=YnfHLq;XF#|bV9O!8Lw;2Q4B@rv%=L*pZwpbu&TGu7gBq6D5>x}^)>Z?o zt8FG+4RA0*FY9gM?FNy>Tl*e2c-RmvCT+^Q&OJrw zfAxCuqoXGug@D6m~y-V9Y>RAELQhvx06v{?NXL|G>; zxox)^LvzfzwRHdmwhIOGE@aR{(a5{`Z`!mO`X}n$yqnw&Lz|g%Ys&!Ep@qu;MkCgY z$9I%D29Y|WB{t_eU}#fwLKVPFWPH0yhx4)!G1t7aSj)RCxTUSR4Y#%mpuo6*Hosch zQjqcC3Wd{pLwNj|l5MBTF*L`9TU!KBF!3b8#FLnbM<8^VHZ$>v&1#QaU#DPbGaGJg z4M2fy0d0P@v?U?*g__y!deA?W*Z1P0F&Ns^rs)!Z{g|7q!pX@AqPQ&wCVj{eMxz3P zCYIdV5`Y5RGwViPR|0T2vK_6yS~~?I`Ys)_U5SkrQ2{Uq8IMhikB^7Qc?U9lDq6FQ zqk^55gbIL@(KY*U4~+*RM;gFMZFLNeZj{;zfP%4`2*z&0us#-zUGBKvNpzz`R(-Qj zU(Z4_1pv9V1pw>N!UX{J(2a5;g0bDV=+)`Nm8@c+W>%=yD5L+S#azS`ljWYt+Dk2z zc!mDan;FBu_>yJL%hZ>aQM0lV&mEEy88tN4-P_$i+|4uG-O)Kbz}Yc8*w-u2ywQU) zj{eT}fzCRC_6}aoI*p#9W2p1*zT#ta+nAp_#$dvfRgugTz9i61u zWTpXSY)m-nK^f@o?Hmwj@6Fe7a(1x~@(R>(4D$DP_6ZFC{m#F9A^868-uC_>I-brU zI)?EvMn;+XQlcD{)0h`>q(wR^hdS!-n}<4ToeTdJM{Oa>QOz22R5K}&j%r37l{#uO zK2aq_IjU)6j%wOMq@$WrN2QM1e2yw5A*0r*lXz}W?RV4w=kR_GK>@*Df%U&h3b|zc zaq5TE51Y>q<;7*x^z_7YLw@@q^L#{F$)54JU%yLWzWD1M>W$PJo6Q^R;{S$q@qZU( zS*5c|`AR-YDvG6wa}|RX)fLJWmMRQT(2;*AzeawbJkw2(z9<8f0m=YnfHFWCpbSt3 zCJR79%uo6R)QUDNZ1H^ku{(J$&cGFGQwI0txlT!e+c|A2W zPWNA`1Eqei`X=~AVQ6#nrc(eq@QimHp0V!URJd|!W0d6$dkjrV0YD=^HgxO74*^iD zcif@Zi^e(uHk|^X#p}rrd6#%U5iT#NvR@i;AG2+0&aG_(SQpV1ZUm^wGcGyxCVkss zxboU%(x{XNEHpU-K!ewF>}aty*K**LM4Q|1wRd7@QU-tyuV;LhB^M{1hntDpYaa35>7r$Ugc4G7DJm` za%)=v3ebG!VaTiY<1-O_bzV#EZgMxzYed5x|NyKxuH;E@2eQsg_We!ApUP9CbvOWQz~ELkzyY)E3uxW$9Y}A z-$+45&BjJNcUWD(-_?m9&IJ9Lh<_l*t53M2e~7Ppqeqp67PcBchc0o^C9Y;$;>y&Q z6N*p?d8H}85myx)R3vz+;1uep)KQzyQRQ0-MFs-H`vp3hF|h&rfCj%S2)Sj$cc?c~ zZ)`?yl*PT#w1GF){Vv~#H`cvFy^(rj^Lb-;85y-+y~J}9@h_ZAuBU%^ptCn$=-Xud z;g&)^+QdoJU8%b^pS$YIH~pQ{%)gQEoPvW&O}lEK;1uep)KQzwQEB}@(pm?7k1{|R zpbSt3C%@pgsl{>0#< z{r?U;b=hT=)6*Pa#N@NV`=mBvXwv?Fgl_MtV-VK^5>CzWO7LC4LX-FZqltgWt8gkZ zhGc`X;q(98fT2nI|2=r>>fSv{I7=b9OR&=h@1q!+wEsVtr}nWh_tgAQNa$j^YQu88 zfdHumKrl~T=USJ;dEt=!XZ6rq?QAgHr2YRM$k}8*-<&iDhP(875It!e3r*ht-xtkW z^NZXoCm5d7{zK`_A`DI1|Ib77F8^_H)p8hqaf+R9NpMXYeWU4E6U94Ci5P%P164nNdFJk zlGW)2sXdkszN)()LzDFXSY&&?o%X64828hzN7$ja!e~bSTQ%JQ5Zxxubmg`$y94(xvOK8$*Ka9B>?k*tyb>i^e9Uaz74 zpQ_}oq^x*ZagJh;qME`@g(V7M3LWL|%de7;K_}7|Wq>k38K4YM1}FoR0m=Yn;GfQb zU7PwCI-lRL_`g^_Pu=K#&f^YBklD(AZnce20AHm1kL9WN4hk~vzY5Z;j&8PH?j!sS zU4;A}%2V&WI^HMz31s&5jQrkz5QZko|7$$8vSn%OCEr1Qo8ihZ`#A89*O6LR>U zxnftmgS+g8jG7ef^-CXMBR93NlWusI)s38Am#H!z-DPP5@^4l^#`iU517%JM!$s(S zL|dnwvpg~^A$@Skj=|MaS!ksHt(va?&)j9@L)|L&!_-8(yMMIYj-g5VAGu2p?H%RD zZy?)$Oz3FM^%$BI{~yd#*I%EuIxiKnrQEOFdXUCKll4EcZ7kEq%u5Y2dj6?f63%)) z{I35`_5VNp1B}j(GC&!i3{VCr1C#;E0A=9cVt}Ckry?1AGFNM26{OEt@ysq+rQtX9 zI{k0WZCx+P`~|(m{F4|-hD4Q9(A~R--68Y(*=f_lp&?pi`aerOFTa2cg#(tC zJ`BUqME!q}r*?bA*q_CZAHj#Mx?V`Zpj;K0V(>C~W`HDyCh31Ly3F|@e}SGL|&`2ttv*ShK;p1Q91 z3hBAMAS1!s*vv=kH?3q4;8|c+!f!tv|}y!{MP?c5(Q#1Kc)B*1&V%hnhxD~-&f1gC%mAY&5xvP<4(|vr+0~+h&`|GF*!C8NuLw%L{YV-MO7gZTG z3k&hwc)SImOCz2tIM%jlKNXxoy_9-s^Lc4^m8QP{Sh@Q;Hu44F_u)28`)c6tlc>8= zcWpLzt*ie>^Z)-#5|;j1$^d16GC&!i3{VCr1C)XPMFt4*|9qZ$i(HTV*aFBN72vu^ zUubW!$oT(2p1Q#ayXro7A!GQ%xpRl%JMc-{|4}%&#biz0eSDbWzA!#{qOx#w5bYiy z#{bXq)Q**Hns}=VWb)g|T-#QK!AbFdH1yU-hQA36hba#aXK3~OfT2n8f0UAKbLUTo zPbM&>s{ev(QW7jQ3T#?%mFfeVd_iE7UmMs&Xz}yk?;UywX|7{x9+oF#Xj1$ig+(p6 zPLD3?!j#mB?Iw0I$IztsKME#G&Hqra+8Z)1esK86eTAV(@qc8Sr#EY|);!3tcYOOb z_YHveXg`r8?{}F2vnmN5MjDs}KFVmu4 zea6tl`2SpFJbw3pQ67-3sFgHyca|_*MEoDc;H97G%bxOrG}8=^s)$$?njHU+;;G9S zEx(}B6Q)Ey8!f#{2SbzM|0r!jHUCAxDRhTF%#?n#ej$b?#s3jnexzHL?gB_3yJFUz zfnqE)IsVVMs^R3xd(|Or=HcYiTz1w3-T(i;_+doHOc|gIPzERilmW^BWq>m9UuS@z z|MPh2Ee$V!Nm{5Ihk@_EfLk|o|HX1bsrZ^9s7jcR$s>tR4qIv4ps)?&I zO(3hC%AZDW*qf?F=>PLPwUVlW*;H!-WUf~qQXPw{Xp;U%s|V2w zi}ZiKg&fVV^A{<7#B7uFKT^@whyAX#&4Kig^@qIl@ZAF>{f`9QP_f4u8s8w(NUZWl z4|dH3D_fHON37XoKkA#I3R!RBwQuY#VU3)u{}DPmV9>{n`1j zuyGOk9}V5QAfkO_4@fJ&+{gCmCl;El{~6L1PG+t&feeWiE#rQ;VrY{7kL0P_n{>UQ zWDl94sv9)F+``Z#{g3W&?$;XK?ROx3Vvp~=|70!MF!lck`TuC-KADlGWiKFY{t8L+ zb1hi5ss8`3e{xU*lmW^BWq>k38K4YM1}FpniwqF-|1_SuhugKmJ)|Jx#Ou3%Mp`xe zjxAFEGuy#GFFReK1Zk0ht+yQS*bpsp|38w!Toe5RirJ8Ed;V$otLvP4qIW zXIoW4*4iu2Uaa=T;3WNzc9VOmo|BC@3K`uVC1e+L#?U1Fk5&(oLT{ZI-4`;dqL(}< zv1Xyk`X31!ORp%~vv(moKiT)@)*=i|(*FoO_|3xdJ2jBGYS;st5l1mJN&lmf_X_l# zx%DDsmu~)KTFOq@u(BoTe{{h|eE2xcX#`}c*!E1VnU2{e>3_zy;_R6>ro)s+o7YWC zXWJ(0|0JGzd>^}l)I6AS;=}ISC#qSt$?<>WY<7=3Pw5(fY#)rUne_-mi_rg8p?3Q| z&xiCvHy0gimJ~&q3!>ux9=euKBxE3CWxT|uk38K4YM z1}FoR0m{ICi2;KC&*!Pjw4b@I{aeUwp{4b4=lF)-vFr4|P1E)Nkcc)iR4J7(fxIU} zG9`Ahs)LKD|Bt@kOX*aO9%m0z&%6HIXvq!-i-`YU<*6Ol=~a|{9wwhST&`Fe&~WIr ziw`9IkBn<5uKm(Q5%RBd`cIB#rA3Q~|05l2dt^$V(03>nJ@LSqW5-#xMe2XCJvxC` zRUms+M~}d_h8UWp{}WL_*X~i^H9N?faB`pWo^BYLr2mnillYW$;Q9v0UFBfAesCs+ zCh32K=6x#dFxdt&dk-F_HBT2qlk`7x!BeN5T{{CZ0=67iKRJ|zChLD>+cM?-Im>9s zoHTX*tU_@NO^W}c%!L-+#o}v*L)NJ!J=?5S!q6oBk9rSCE<2~v=&{zuLxqq*_gxp36*p<_TthfypvS^uMj4@Kvr$+7z& zEo`1r{8m`8{4zk8T{n>f5=tk=eK%mqXt=nY@h7?z-l%j zBL0uuMW&m((OlGBk38K4YM1}FoRf&Xm=2>O2- zPouZ(r}3j#!qJxMVV`(E8-B^pcYX`4l zXp;U%3Lf4$K9*VtnXmpZx>>Q0g(mBNWP9+t=s@QJv{W^VGpUgdekA>mWU%Y!(PgGp zkg>J0e`p1}V5Sc&CX-q%onB&DR@%-e>8NLYZ7SpJ*2-`Roo`*6oxjp;!4(+Z1CeV z4?9%BpU;$RK(6Zh@nh!6G{|h9x^VunV;Gv0|BqG=JkMrdQl1W(ZC1Cs(no`ZCddEL z$cOlMmy9|J=~{F4^x_CTn>e-&>i@47liQ3s00bxoO01Vy@xT4BrlbF7F;Le9z^Gl5 zb*lNk-u`$|)!#?kH(gWp_c^qrDlMtnY$a7QjoLKntFC>_IJmF=KC02Qul_!V`YQF+ zX7tt81Yb2ZZ_HN(N7YHbDmaJwD)rUo^Hu#;GHO;<;<-cc`u~oOjnq~Db+BF2p8D$) z>ZsIFo6k{=+BW%6MPE7j_(S#gQO%}Z_4hf{SE;Wyo3GOP|NkttOlLzGpbSt3CiLSFtnKkSH6DUel>OT_N!s$jcofHddY8zfy^~$HSdiy!O*1mKbm)&r>ddpS0Hb= z;lj{@C>EL=|3{_hoZJIPO|^ualh4wgZD!Y85V`-qjm3~5(HkNATgyET)y0@?jtN($ z!MqCv^DboOjY83FJa(yZ%OLMaYU*j*aTr=u{=e3p!~Au}A#2CdouOYth2bLd|B-R^ zVXgb`*@|5CzIL_SLKd1F|3{>&?eBGH|9i;n88t=bPj>DBs?kY^|D!aS&ixMLN2@}% zgubQOiUQ0wDgKY15A8$dJ-q$~vfhsR*r$kJ0ljz0fpu6l;!q4#& zAh-2|O-cio2*X9_f95{v-Fdv$K39T)djnuR9oe?+>F24^sD|H(_X! z{zn&FE235QSZ~PjzP?kozY~Tg>3?*|+C@y8w^I-D9?Z9UH_n-bCg=a7^})6;l8+VX zL-yo6t7-l5-2+trH+ag^Unv8W0m=YnfHFWCpbSt3Ci@w!4e79>M!8;)J%3rBJ*hn#qDAiiN5SOJHi1*i=fRYTKeyUVbjQ#{ z{ePLKzUS`rhqlvUPRDEGo?d1ZHWv~9kLPJf4T?-oUx!+g_f^~AZJxdf49$gwN;nsgR{MlW+7q0vkC=|D&~s zilv4DPKzLW61dGPs$!wZ`hO5lLwuxr@>*rcED!Pium;Z_AnAW3WWSAc%Qa4h?2gN` zZeCiB*(T|KBxG;MI9)xr4Kl+17!8 zfHFWCpbSt3CnIIAiAjqd%F}4kVbfs;C75Cq7oYK}pYS(y5%GVNnf#^T9?vTk(lw-K=MUMBp^5tc z8c%(?!NF)Y{& zvd0+AcIA|^&}99O)Z~>ZyJ~WDA?wmxv9WX8VrY{7M}kiNQ{cGdu`s#Kuu8EH-WZys z|B;ydxb?!%{Wl?}&ry}d8!9j~N&h3;29F-=j4Ffd(04}HirKp+tZYg8A7xEE>{PL# z$2CY_-}{iX;vSZ5vi?WTCh@e~ea&ac&K>#v+*)?#g2?#)cf~I+jxK?WSH1ejgr3H1 zlk`6t`Hh-W@4Dqe?x6`13w@L@G*SOAMt7N6JA(mIkUwEun9TkZVYq1h|E|rtYc5Yw zpNz`lVY^?j&}99O(nWq;tJx=G4l}jCoc{9ZHijnY|6x20rKFDmW|fc~-&N9W*(VH5 z(*G!J;`9VeBAChLD>+i60<{DSF_vu$(wOcz#*PMZIZJ?QBH$^d16 zGC&!i3{VCr1C#;E0Ahfk|8sci9sT0JS4%+l=;QhaJ_`xDe?kANTTVDHTk{-pRLi>U zsA7jj(cWSXw^jY>fu9OH zxPP=hO#l9+;?*}j3{KMjC?2e*`p{O)8#15zg@+ta(oFnr{>W=~u4AiM_vtp-K85UGTO)S14_K1ygRD^7C_8spu$! z-@=63vVpsl2;8NFaTmn7qwd@2L2{6J{#ses95RK0t_OtWr~c~8zdz?_}$ zVQEX!|0s#Rt@q}W$Id|7rO7XbIbFtV6ZQWro_dGP+wZK%g=`zixM|~C3d2R{e`W{& zF_%{U10c)n@?kEp?h_I9|B<_Bj`DF4({Dl>uk%C_PGifegk>joY5E>#sB3th)Iie z6GPSigC&Z^x+zA>osv7uxJ|=XT@7|kO&PVGJ;igQ>*}w1yZeXpor1!H!=2n+!!3f{ z*#~rli&$6X@*f?BD%gg*1BJh?L0Wl<<`JVX->?&h}pa@gIv# zh-rquqN5WNnGzYElo+3oGU6`{C^9uRIWq1qP6{y=6Pp?znG&Cv@RvrTqmvYyJTNhN zSZqvqWQtCpySH;dpuIO=$I02nKFBLj$1$kc>R4vVYsjeS>51ot)H-K?b9g_-A6@v) z_6>jUDCCk2-=S4if8Qz88|(b^Uw$m`{oTFo{X=v-okMgC<714BGWFG)ELH99?TQxw z{&lc}kbkJrq5qY4gc5gb);3UyTKsLkf6 z=-X(AhTlkqf6vs^YT~P@y^p={SJS!!orT@E{;O%-+thWb>o%L~W@@%?)H&VRy6M+} zPK`U~ueYglQs-zbFHg0m=YnfHFWCpbSt3C#YA+KlQXKjl{HTWznxJvbrY`!3p&Hok2ZtCrJTb$UcZxyN9ZZF2p8RJqW? za>@{26v;lSHvDtp3Jgt(|D%v>%jW|Y%+rPJ3$l?jCGs#dDgKXG;|13|Ncajlj}0o^ zE-%8+r1(D?xvbX{g-L55(@te_yC_x~yomTe+Dy*5zwgrI{*Vb4K3!wj$tL9ZKN`9G zuF7Q>-EDLHE9qd%< z;li5^#8$ye&Rmb6J%QkD+I51hb1(+yaD*EF@fnIx#REQ*gzgD1dyF!jak3@2%!$dJ z^Pb;zU)v6j-0hSRaFo z&niiZIssmxhqV)C;+y|1Ozdi#{|Stj)S>-KYR5heUYZ#{b64hpZ|afa^V^JA#+j7| zq3Zuh0^>=waKxP6!mb|`Mu7YByb6=S3otmx#7?T7IX?k&es#<(MAl8x`o53@pVUF? ztH18T;ASS=7WGs26HMK&cIu-MI$^;y-wWWi<@22p)1M2YYc~(rNj5Mp7`tEH*yq4Y zlWF;B5);7FP{H2AwLb<&s|OMdjPnJ|@tLWo!b{7Sqet}Sf^VzI<}LZGr7C0``MY=n zw21(1!k`DhOLLFbOHTWP_a3iZ9Y$?r*=AM`nE1b{{DV6E@1iWLbXF-}$wx^=u~c!c zVz8pRLb<|Hg)oJV=oI>*3{VCr1C#;E0A+wOKpCJ6PzL_)3~*}~0>o>(stGRy`0`pm zy>(q_z-+KCQWz>#(`fU7+I4_BwD3BBH$tC%R%|#EtoA;<@8P>mSX|Tlfdlf>`QGw*|CORn|x5a4p297;dKB$LSG&}s@F=e++jB8L@_&Oqc-`Vb{RlG zx(eeWX#GQl(PRz?{b2HuF|WaL^xjMECuA|(91CvkDuBRtVI7)Z$eRUbg_i_*&Tsx!yOx zGCy(s{@JZDG)e!vBlO{LTa7rdZnNN+#j9KlP1662uN``HKfa>Q*ABX}&{Y33K1N@Z z0m=YnfHFWCpbSt3C;FjJsuHcqXA3>y?VCrH?F;et0jT(3llp%sZ&j&w ziViu7#w!cpeNO79JuALpaFYIS$6Iwp`cv4oYF@hw z;|}+H6_24w`X5dFQuS;p(>3r;A?;9Hw{jMmtp7t1Z9`{Crzt?q+aGanqr)*YDn8hx z{*Ontci61@&<|dZS^2(?swajf>Hl!vs>|BOgQaG}JBKl1hdAs8fMos8TyXA4pMI+F z#%Ok%JbirSf~5ZkA=}rlZuPhUZ(S6)F(#~9bak5#n$-UhXx=-I_wE=6Z|!PMl`g?! z_$2)w%Ug9s_2W(3JMi{)?z7=n#aOJ>Za!#I|3{&Dzd4gHlbimYtjYY8$*6TT-ytS*SND#}?K9r}6)P(374{g)%@HpbSt3 zC3h=MaZF2nIkD2!c%N;3T zy(99$nx7sRnxy|-knQ(lE-!fqRwlBOFABnF z5%GU#H1Drt53SUOUa~chvf`33G)e!X`2YJKG7pq)g3ZzKn>)zfz|bW9&)nd*Y-bi% zg7vEDJ^D>#mC2y`pME}21}FoR0m=YnfHFWCpbSt3{%H&}t^X5vgDf^Koy&=YvVoD^ zi>^);78fG){~+EV%LRtxPjcbLt_9mrxH)xPz+#fBs^E59`)2s~VVrY{7Pvi|UuQ2^^s1nLX9`YF7husJd#YHW-P3r$4 zyg^o*_Gsj!LfIatSxxhN zplry!GoSZ&z|f@p|G|j!7Gb3U;c(+sRqh09cGiTIElK~Qky|EgTJ{dH=5~qGX|F2F zHaY)49?kpxu^IjK;il|})7OqPT70lc{g1AgWr@TFLr=KTVoTzhiRzebQvUxiH1Byk zI_^w{vipxK4W6@Ci^%!^2yNc?TdO@s;ktWo-F^p`2-_B+|IxfHm>mrMaAV8F9e=E- z#?U1Fk8E3|cO5q%1j^djCv9-Mh@naPe>iWDjn#!hy?D40JK)s&^u;XnulT>Tn7f$V zDOr7~EQzOz(dZxc|K3Bet~;Q1XBjn9Q}Ntbc6UIgrHYSL!}frJBSww)xd;vyyiXfl z&_)-{*61QrQ>W1`EOp+7xu;dI0m(b9f~j*-=WIUb)b7~m!_&R)QP`lj>hB}^jl1XX z_o;tU|7=G8?9ixx3O=eE_~-8r_T&SrGZP6X#{@XfQ%`>cWuNzO^XdH!}y+Bf}INA>@_Q#Gys_wTKB(6cE6lmW^B zWq>k38K4YM1}Fo6YX+K*|N9iRQJ=-F@F<2}J3adDdazD7JjlfV&70o;&!C6B6<^pH zY*X*u+R`2m;1lEjxd=Yq#L{IVSg)@#EG&4088*)XLuY}Uy6|Go%4YoM}?7J^u7yJ5)XJsI=)iCmWV+a{fQV+PR&p9fH7a zpvkV_r>y>e$Tlhd&$w!O$Kd$=!0o71Jm*78%r+_ipFz)m$nPBw)-v8o96#3X0TJ0BewhcPK%2YvUQvBZ&q2arWYCq^z ztfziAyshxOY5xD;`oT>HM;V|DPzERilmW^BWq>k38TbzvXj=bA@&@@-O|MZlh0UkMHIM1zgTYQPN-$R7{M|%(a2fr-+v>q;(OZ&J)O0aDI%KxYR|NevLDuqx6CR^ruI&ECdhfe?DX#@TSLA24_yh;3WO8h|qS)RT5TUXYKe$+r|Ih{{KD*o%V24 zR}HYgd#sz*AzVe1_W%1MbieKgw*P_v{S-vJh|1&q)xW|0=Rp6M~ zHgb936=B;5P165?2t9CNPPhADFZnIi)>a8alj8r*2;Jf7`46qYUUNx*y{@lWXsZAJ zjt^`)CdvS1fHFWCpbSt3CplSU-m^YSx;NkJSW$@UmLl^V@nZoiwg#I7O z8|!!fY{I(}@MJ|B?ZI4jNY>hxsQ=4(V?(^*m}zTxl2>wcm!k}3oTUHTBI6k+Zp>1K zDsQc_LNRtdxZ1pEMOiT!nYy0r{w55AylQ_FrWRWgn)um+S9i(7<2DNxexJ^&$;Qwkpy8xh#f_a5Th_Tt$=W z|D%!nJDm>Rc?&ADigtA6u40XxtpA7e#`^w9oyNZaRcjt>YVni3RfU}YkFJ+5wFfp_pxw{zta`dzJO|+6GlubgR}_KM+QX(ErSBGHgiR8)>LIZaqA4=voX- z(*MH{de&i;OF!Uo^nuvhb=t{7Q~m!Jo~sl}8K4YM z1}FoR0m=YnfHFWC_&YMtwEpK8wLh6(5Is8;9M9^NEttZVgXI1HjtD&>Z|WE&aOjtu zygPsu7DcNEME$=6!K2ptb|?aeolkao-eK>_wzehde?^2IJhoVy3l5*fW$lXD{ou*^ z-;bHNMar@b;MB2T)ZAO_)CIEs_dw|JHR&&&fPGQS=mk&NN|qe|_eAKJL3w@Uz}Y^D z(>~#vuqH!m52XG74Cx~;j^~Sm{p@E(wj0^=CddD65qj3aqeD?Vcxh?Hnvv|14P^cA zjnLaiFY_M*&chF!9CxxSYvknkzdJ(Dc)V}fAK;WUQllu5l?RV<@Jadq&ItWpMaIVq zoOq*J3^5ss*(SyR8AqRMZBqOm9Lkml_tqSPp-K9mnRlvBmg5qze>l8+`^1I9Xc79K z@wEw7rOQTv{Q}keWBD!^nxy}k>z(~lQf4*qp1sp{EoU{^M7Bx#pW)oyGDhnHIGszr z`zeRjqJ!@L|2saf>6j=3lmW^BWq>k38K4YM1}Fo6g@LB^e-dw0e+8YK!U=Hm`Bb-< zPkV%=iHP_=LWhOVbJqL>H@WR6e!0R5E;9PxoY4R8A#YT)`E;JeT)3^cQgf9rJDN<^ z|Lu_Rkg?0B4TGB@zPBdw46&h;_WuvzjS3y_S2S=fl$EbA*mo_8g(mC&L`2)y)q~Qv z!i`hM9xJHq!qBAn|1e~`cuRKo7jQGs%C}VntDC$C{XZ1hz7cw^=nCAhPrPui8%xn4 zG%5Z+m^Z3lAp9%)J?k=B#;>^*`fm8pe13 zmQo|*R75kuns4)+mx~_iuhB9haR^qus>bm;5I`PB3-TlM8`h*9E2Rk_jI{JFM zH+IlOxQ9h!r_d%VzdKpPM5aWBCnd%wq>RwZtCH zjW}wZr~Ef%e|~fB6RG`@4JF`-kXwI)~^O#>W^LW$Nh3sBt*r zxpC}1ss7F`;b#5~npFjd|2ot|*h!fuFyAQ%dG@apsEbk;ZN6uzjxOe+0et2Y-^tyz z-bIBEb;n&)_yp>r)J2=mMYX#$+K{rYg<8XI)4z{&#T}G+hW>t^`X}|z=JQW|gGRrO z1~jId8XoK^?4`eSQ^QlJqf$q0HbNxL?Wyd&=;Zu=K070#lGV3hN{;#eEjPIi<%K&t=mh0p;(PvWb<;ExRT53kuZ z7f}4)lq=I97G1*J3A}2*5~k(?<7|O1WZ$WP-lZcNX8Cvo-`tcdU61A`)uH)G4B8Q) z-KW^9eFVM3Lrm^**jJ1k|7UDRC{-Wq1^PkT*N>ae&K4oZ|2+|!$KUk4D|A(SFfeIs zh%o0O;{Ux6x{p$~vScs_NOG7~!rJG8(4_c3Ahh$qoz=ErFreM1I};ycwn_1SX5`FW z@&ybMJ$n~Q_GF=d#s6vj|G(KYn+}IEKpCJ6PzERilmW^BWq>m9pEA(2{`cm!==7vQ zyZaF^xjG`K(%D2<9*EHY4EnJ#ue~|+c>J|g;qF!pP0atFj^OuQBPtd_w4+?0LFmU4spr>1*Xxnlj^|mm zCPnCfCxkwmvbXvZboV%8ZDy#T-~Qi(jc@kE{+7?efdJkcvgSRaU30B}I)%^L^Kw+210umtIS<1jSGoU2@~ zCjI%$oKX#aKBFebBlNE4>lf;Q{?*+}N;g?zXp;VyXVCXUEWd!R&q{|7S(Yvm5&!o@ z=)wmzx3j>&W#E`yTl`tJ$@3>^S$qBS1{RvE|B;H0*rfiW&vh_V z4h=bY3^i6BM$ud9#RcP)%W-hqVVd0BJ_X6_Mo(3(O_`%+U&R%rkHJ#{zn=- z;)LqBS%uJ5=|)E48d(fY(*M>7z2waGtA1c;CgVH7ER}^O>wlzVBX(VqNZJQRN006L zna_%%Gxh%n`Ty<+y)f+lndzV}UGWDG*!w-m`TxxI-oB_`yEaHgJNM6Wn}m&=6#r+g z*tX-nI}QSat%}?iXIPmkBJ@9VlWo14Fs(0i7u%84UW}D{AVUA!BlNn=Wf=oNe|yCn z-K%SaM~=`W{cnTN>-SeViGjhkhZi}ci&$u?|Np~hDuq!7CnD44o<@IK!pBBVep9Ivl6s*kpzD}*rVj%)c=zaeA1o1 z7JM+s>Ch`~`T0Hc$i+TGf_53?;w|4&jLaQ-56iyI@ozBfzBiirO+>mHH! zW3~(dqkO&F@3Pn%KFIo?S$kk?qgeD-rvn>wuCPW<*8dFWq1roNhJfK%55G~GY8aXn z|7R5Zq}7YWtU*83xS?_Cpg31Mc zwUrAP1wS@H{B0uWmuS>%&6dW{r2YRW$0j1dYo?Y47%W+yTWNO(LzDDBGx8C-n>RfN z{cm}Tp00brLR0d#Fhf8E-BcJ69fHuae~M{t1IxwE>G>V;5Iiydzm+jA7b1TP zdTkka`BBz%VQqpGJW2m6A@uT9Yct=1-Kj{`_ob>Bnxy}u5qjU(#go&)=E$=j_olNl zx{(gHu;eP#Pn=&;H*tOmGx0tMy(X^n{4HSph`%~1g|(|1p-KARAED1Bo6K(mRxkao zt6gA|PS*c@5jyY1+db7_HGIyUNq73Q=1t!JAA!)Pq6h8X0lnl^E`3}52t$+N|NRj< zyw*R@k;rVI%14GR3-nX=(Q z^Ts=@G(3fNSfL$Onytf1W|yuqY8Dpax$(j+RO?#$2>WS+W1Pm_w80tFOR1MOqn8>r z>ZNskP=&p;!7C>mH0OM{Rh}qH#xUcnWn?>Zr};sI>pz|A}O!MkoW60m=YnfHFWCpbSt3 zCT{OKZ>d*#|5;hTnoKS)BZ41d&RO%j{k=uboT03(F?#fy50LbPZwio(*FM# zgdTP3sQ7iTx+?3@Ye7E@P0IggY|kl=3YZPO)T+Fd@%OW$axONeK|LAlbj-AOX6RzJIi_5x`iu>~ zLgwZ|sVarPQdK4-Tn;P;TlECnS8EqyXj1&2Nt2oR<7 z)0TLrL$B9`@?s~uV`y`0u0(^8^93X4Gb3kwZFuD6Sqfky>$yqYCj{U-2-OqT! z_|;&e*8f|-5Y~EHZQ=T0XgfQ+r!BiA zJ9+=V4?=sp#p!v1;Rk0ks~;Z1p-EBk{|p50>*KQ;)&DC9E!hy;7lV`H|MG&N>w-bx z3A;eaSs0oW|93#>m~NHd-N4{v++M{MS6OHj7bV31dn2^jl`aayz(DP;3XDRBp8JqfjG@gexh)$+lWQ2iLUHgK zCYTHe9rWkbGI`K{IWl1219k+T-2cxLp?M&Fqy`L^@a+5cy1=qcj{iF$bcon@lMc{z z%=$rk7wj-JY5zZS#Vm8mU)P|zhZPgn4^G0+r1-xpLOb<|?qCMp%v)dUk^KZio0)Q3 zG+=F}fVG(nYjz0jCYRr?2N+~#`TBdYx+aQ<|GOh}=;c34JfZ9J9_fGXcqVL{+5b<7 z|12VW|S`|My>eJkvi+8K4YM1}FoR0m=YnfHFWCXjTTA z*8fPzZgvYSeR>ECKffJo+jX+AxDb*5kHp1hC%d#7P0*`)7PEFSyA_>?{QoorcQH9^ zZUu&GgS|_v&SS<&`dh$I_yx_8>|BGnXtx=9T*>Fr348e$amjW}B4%Z;8<65j*-tg2D2#uGKbWEHqjF zGq(NDnLj88-7^Q2EKknH&?Noe3!yE>Y+IuUT{oK-{gJ%~LzDDB(%_rDr)|8D3uUKI>qM1|!hzGtC-<^QY7KP2n_0SY?k6#AkJ zPzERilmW^BWq>k38K4YM2LAR8G`#?TqKjKDCtaO70t^FXb!YTf5Y`7G764FObj#Mn znhYb*Uwb&XqQf@~%`xXH*GEFz9WzWb6e+%r;5?yCK_$_IgDg0zIo+5sqOT7MiU8k&@lAi<=!X3-r9_AL6fJ zcP=FF|3?x0Eqg7wS~g%{vvlCS=Z7)dB>j(c@RoG}Z_4t((Dlam*?ZZGE@b`BMd(wS zF@xrT;gk(FgFY?AY?JiAJwh)&ZL;7P7`*WMW|tJqLX-7BGw;(wXSt|?QPk{xTO`FW zG)ez^AoPNn-#Ti6!IjpAONUIt&?NnjR!z1XA9Bt?84NYHpX($RiJ?jQ-wUDt9P)L@ z12EXUE-T9XhcH@%{%5XO(euL_euCk&IY+Kc?uemD@qeV?x9p2p@p3Qd_2uoTni`Fv zN&26M(3{^Wo4f}7t|MKg48vIHTK%usM@(AGPE2l_Y#XVu689Cu&_66hJk$`Tt_Hgf zM@Egq5zmclthBY0Q;;L87%OwguJH<&b){ID6KH))THmtS>RV>2naZg3=pmjPDeR(k zT>|*O{ZsJoUX44a;9csO)H9pWGff)x%(@3*y=MykZQHnK3f`rjNj^Cf zy52Jd|F&t|GX?Kb&!nE&e4eRq+-O<#y3c1;S@nhots3{yhNnt(kaT|Bl`V*+5Qb%nzN3GlcPvie0Ry^q2lmW^BWq>k38K4YM1}FoR zf&UQ(nvVZF9(ks6b=}jhXm4`C{A-oK@MZk@x>I=+z$Bqv|C`*?M|x$+Hsg|Qv9E}Vr6L+ z5C#T^O=s+?vBc1%`2VlF!~^sXZMV&zks^#15&ySCw(mJ)k1+)Ogs1bDUHgflN%4Pl zy;Tl9Q|iAT4ATzZt(?xq(4_dk{gG#Ky&V#cs(=wstkbdhvn+Id{y*LS|3C6Dr^BQS zPzERilmW^BWq>k38K4Z18E9Jm_dfFU`~CT$@~=Sui+YczV&%f}K!p5vLFk8J^YnUw zZphx<>C!3~n%Mtu3W7iIF`JSKdY)B#&!ywd2Z{Or2%U2#YluGRRZ8#Oy(k1TPSXDl z2wnR0)lOs3Pk;OIbhr%*P1gUe2)$?Ep|dHV$F*>{Fnu$IHn-%;H`w=3+g*)U?XOZ> zgAbvr{pJ^U0DZe{JF4~5F*Hg4dm{7|^ONO+q06V2-N*Fv!_XxCZ-UUb73Q{&0lhSr z!)q7p#L%Ste;z{rv<`Vx00uEvZeF)&!9tVuKljMfA7VC(6<>gXS;C-a-mHD_9J2m* zMrh^5>-WqBy=8scU)eewvrW?f=4j-fhgyyq2fEzcs=eI~VrY{7w?ejcB3JZrgD%H% zIXN#``yNE-e}?lfW*RDnpqDdBCbNQ7u~EeSe=meCiGS460Sw)xZh>Zn@W>IGr2m}| zy3l9I=doZIkrnZ*pEwI$um5TMpZpM|?@|UR1C#;E0A+wOKpCJ6PzL^27-)I{!1~BD z$+9&wRcC`y|B;tZy}2%|4@4{g^g8maMN0!?Wg9SD`Qf(D@*oUNivLeUwy$2=?dAYR zJABl{%uO&fN&lnxzvPw=mmhn8p{8;E!g*{ZTdRu{8)$G7=4v4gZo+8r-e}})OBWWN z1O0yIyxdJSS+>dgAH`%PE4t_mn+|#(+l(I(%Bre{YVeWtKM&cK{&K2C5Ew+?w%NUM zK4zO^$(5@gIlr)O3?*^B%ciJw=ffoRQn`F zUchS)lJvh1LSG3^xsm{edq#P`liR?uP1gTt-jWX-#|P(w-t}l>*@9~rnzaAl6|tsi zvDCp449YztZr=HS?VWdAQ&<1T0}4SD6@3)jI#2`xxDXKsauRlA1q>ph1f0c*=tJus zEh>T*sC2 zs+~d;A|Dc2{-FQ6zq$#d=GFg$s5&}!_@#}+DO*{0%Ljc#Y9sldsyl|5bs)sWW%=b5 zo7nCF-26|~5wq^M`H;vq?Wk3I!x3g|UjFY+)vk`SCOjf;7oR7cw-wL9=db@4Ox59c zf-ikeoYzlm-s`BA8Jm~?1F5>*Blq2Hh-}!`Gxw%uiPZf3??cs5%MN_zNZe+h+3|Cn zD2p1Y|NBz4^_|S?TZrSTrfqGr)Xdnt{NItP+n9aQMoFAI9_$l&eJ`Wt=6|a8-ul%E zHIe#;b=li15$T?eyY)_vkzgnUxI_Uq$y>Q||5C8!X009sH0T2KI5U7F#D$f7@dn7kg zM>eXNPn@rJ{%OL+K*MNaB>&T3b90jWwStR8*1=-^OP`vInmhkLlgd91KN_1voVU$= z*l|D@Bj@G+T2wu1vvk;e;-F8j8@$w?QSF7GWNX^gxg8Ji) zKR1vPXYJ`dlWU1P+41ZDAyhpnYw*eK#3g-V-xKjk%-FpAub}GqjGWb}#HDtVTCP3B zJ^1+fUpQrB!yj*(OJwu!5mj}C)m_)q#Km{s zoC$rC88t8e3u6ydJh{4wv~JlT^PWOHgP%YDpMGmn_$v9F$dZ0<-~Fkm=R%qMkNN*q z@XC%u1_2NN0T2KI5C8!X0D*D{RQv^ikjdKY37>MAxG3*AoNdwAFq;Tp0C;-WEtBZ# z3V)s^y23lz+fudrx#gqh5Lx=u?9eMS88vVIe=1cw+;7`;8F7((HQUxLic$0Ozb#ca z&cFT7Od|U|rNxQ;Cm1y^|I4Ymm3QvFZ;4CG^M~5a_(-JY=YJthx`$_s-bvg%UEaGE zw~|rw>iMm=p zd^DF)^XC6|p=#Td8so1Lr}@D-*Lp2v)V%yptLQg7S~aN|M`V9&nwPAZ%BXqyU--^N z*6L?lSK@YZNJzwB7elp?{4bpEPQ9zUeotI`e)C%|s}M%b%l~0i?eg-ct;dP1^`$}0 zSBUx^8ma#a&%3F%oqQ9Kk=#0c>xvfmD9Qhpl0b=dzLkT;0`rHowO^pbr<}`3MVpEm z0XlUy=*tqOjMC8FDn0xHG>UMAMja5Qa#xnO(ciGyMR~iR-38iRn5^AJnv)k}RN7gE zHL7@{FKblsE-)&LYC5A@b(GpUIGCl!6!xv`q0|KR4pXVyD?-ZrsxUBRnFnA+SkZJ= ztnDeab9OdMk1J}SD!fUn{G$us)eGO$i~l1xu%p34FeD6VIz!rbD7URzQ8zz>wrZuD zd<@MQmb=UHTltBn=dkl&a3~c>DZZ?&ky`bR!wTS=Dc>lNX@VRN7E-h+gy6)7+JdPNww-W zqFL2O*8hu?r-V1?UyzzZRy<$$VQzLBGd8dOFO2=^jA!3xkd@Jk%x^lfYY+13|8aC| zn;{eX{6dzh_xUudahDmJSO4!%)%_=!xg?RLPoFil+WnzO&9DFWrRrK_-r{Ox+4^UT zexKZlQS<8my{Nic_WfJ#Wa$@MN8Q}|X6ygO()0a?dn@%dvT|Ab)P2f%%-Fp8zlx6C zb-ZkTFj;nH*4B(U`wZ1a>i_h}!)~9FJ1rzDk9=2uTGL~UnpgktNzeC|YIXdiWcl9( z2S#VSS^q!k7PZg!3lkfW6`8eXX4g&;jZJ;-A>@Ba;k=$IbydT`f&d7B00@8p2!H?x zfB*=900@8p2)qG-qWVAX|9=By*cAjo00ck)1V8`;KmY_PF@cIF00hcY;=&%C{`D}) zy^(n-Wb!S;`k;{s0Dkh6cn@Fq;VnsS?DWYWDa6%eVFG}6ozm3n%`oElx?cVc}hYMM9Vz)#){Ch^ttN(YEr}PQzWN~~V z(cL|}ZT%rjk(yus?uPTt@zM$N1L3rGGmJnzOUq8-;LGXGS7q1s6OzY|r@vb7xVN^+-N zc{H&J+dY6+|EJ93>x4`nU{3TiY*s3Nc*TrOeQs9z8jEaK^cst7C%ne!k@s^uG<)Q~ zh;G<}UFnS)iPXjQf7_bK|CRUxjza?h5C8!X009sH0T2LzHy}`P{tuF;Xhyu6Q|}j& zGwYDe&sTOC<^v=7UqRD_SL0@(mL&I9P{fmo;(l=a{GUq49UA!PD$GSR zMQc9Xcr$?}x}lvWjj?+!8k?X0edu=&8R`u?Pm^4m&e|_mi)XU)*Z=pYV^8({afKDp zwHWYn%7ZD)*u4Btb8zB$dG51hq6^c7Z|IrHsCoIHsx{r$$b$WeJ|wo)__kJznwS6S z^G+BW^mO4VqIbR(=o8jmq~_;;VeB~fxoTUI`>pxX_ah!KYF_@QUem;_oib`Q$@$!I zhh)J>M$OCrf;zQkLGU@UxXr;y&iC#yYF_@Q-;yQx`QCc*1Ieuy(K0*P*HCRF|I-Ao z*|t{K^C{6T-*})wzNk|rRU66w{Tjv8YPOjyj_NIaoFF=Ks^;bYj`Ea2_P!I2oF`hx z)d@3WVnjz?n*VYC{~KITu`39G00@8p2!H?xfB*{flm;bTNQ)^VSW zLR`@0FaIyZqF-xWy%J3H<3BcU=prg?(k~A@Ji8T^|IMXR525TYFe)zpOFp54$97ct zJR;f;hNe`L(g zY;o^Fe*UKkK4F^KfpMdVw&i!-lk>#)qVw}Vtp~?zCbty@Q{Z|LN?3 zgn2{T1~ex6v8qk`&vg+!Z+`zj;e<;DW<9w;bg>aZeJ1o`)V%y3N|S7x=4VGNCi-hn z0&mADMQVQjr|S4-11vkAB>E@yzZ+NQA*1Hye>&$PzS$|aeZz@9uEn4mR`HCQm;WiX z_@0Rt^(=|5-8{!8f!U0jm;bv^_00)meyByXaczbTiTJ`$Z6yB-%*QSZ)$b+x0BPg* z4vW6IFp~f2Hx~&HJ`3EXBRR)vD7GpZFk|!bKXvMWubx_;)l>C}cF|)Siqy#emG`oZ zBLe{t009sH0T2KI5CDO?UWT z4L&Qcv9+dQxGW|TPaiw29e}-hF&2{&Yzpry8a$U&CCA+wZy

`Npmlo@T5LQqdl8DO_qomv&)XTKwK!_F5|17Qnhxvx+d^2ciuQn4;Z>7)HO;|?acS{8cv+W%DWC=APkMsEM7>i)e)`l>q< zopVBu(D>zynmhkLlMX)T`KrFViSFx&O)YEOVdT8}e@!~G+3RZi$KFJ%xTtB=cO9eV z)&J>??u7I`x3%+$cCp!*$w@&XHGlp;olV{^zaU`AN}?T*+3fHyH5fH-{XYSEmfN4c z_C){C-6c@8hm0({?V&bNv^fo3D=9F)#PXm-m|^8v~CoQWvQ(`~E>f$K1vOo{Fk#5Fd5dD{9Q-azj)y)H9XY@LFFfzH zU;pj+iY)P%{zuDas~9z}{!eEQBu34RaL*)*pMMhUc5%3&+DQFhxJ~9vYE<2IM7t^P z#-M|)jG9;f7hYsu=`zQ$L~H$Q;q+0WyG7{Oy!t<-Hn>Io^OGCUdk=Q)eAqcbGf=TkTAiLT&9z8PT?JTL!S(RuLp-P`{7k}P)% zelRqrB{MiL|I-v5(`@XR|8ylfvzITnY}hYS(_GZayH=Sj`Z~#~3X^OYec}oII%F;; z+Vx8|p1Umm=7K-}pU&ux@g3z~=Wn8q8o%z%xC+=1wKet#?8L%iN2KmQBVEb|&Y3MIMcbgw+eh~2`^|1?F% zbhC>u$RzrCdCh`UIn0ss^1txBFMK)uke1}M%G)q=&MZTVq#agV{H1oCJDa5sFYW&q9HI$|aqpl}MfA`_ zDb&gcWtcKb!)%RY`&Tggpo0ZESeUGXMOqVIshzvKS^7W&uLj0w{Gy`-w-)=fbW23J z9xZ(w{)9hG=T8R((+7xN5vkjlD{0wn;T3tbLI+acHO#}rH?B5EGoJd7UrjN zuA{}s009sH0T2KI5C8!X009sH0aFt|{x>yI z{5=o=0T2KI5C8!X009sH0T2KI5U6|vkpC;+Il&Qu00@8p2!H?xfB*=900@8p2!Md8 z2_XNQnkfDr2!H?xfB*=900@8p2!H?xfB*VR^FJu2y@$9fMcHjV4UJmTU!_z9m%^o{ zbE!*^!J7YN{Z`UtqPTlXp-)S;Mpx|Al6~M+c-3@XZ5zm6S)E=Z%UxN$XuI0bugtpa zMSH=qaIEPZ>kv@>X8_tmP1KpHaAp73+oBAeT5?S;Rl%ijsmWZ5`v2P}Qz$_I1V8`; zKmY_l00ck)1V8`;K%lA+K>n{PX9~v&0w4eaAOHd&00JNY0w4eaAOHeyPXPJGPlr}Z>L=V@+Hmars5y# zb>@qTf5*QdP+#x-8heU79aou zAOHd&00JNY0w4eaAOHd&z$1YC&!dCKKmY_l00ck)1V8`;KmY_l00cnb-5`Mc|8AT| zJPQy20T2KI5C8!X009sH0T2KI5a1C&{^!xbV;}$mAOHd&00JNY0w4eaAOHd&@NN)5 z{(m>lBc25afB*=900@8p2!H?xfB*=900{612>IW_{D6dZ{@Y_^XEB2QiXRZDS_IOZ zq6}JC1^KHqO0`C4w${T>(?hLLNB2;cx3QmLvz798LAMojTQOO;l{BZwii{c>Q;tzf zHYQeR)RJ9bR2bESMpad0RAqTr)RK*w3XNK_3ycb*n$Dt1" over a defined period of +time is needed, the module looks for all elements where the condition applies +and where it does not apply. This generates two lists, which contain the "start" +and "end" times of the condition. +A futher function combines the start- and endtimes to timetouples between which +the condition applies. A "state" function returns True/False for an exact time +attribute, whereby the condition is represented in binary form. + +Authors +------- + - Daniel Kühbacher + +Use +--- + This module is not prepared for standalone use. + + For use in programm set condition up like below: + + import the module as follow: + >>>import condition as cond + + generate list with required conditions: + >>>con_set = [ cond.equal(m.mnemonic('IMIR_HK_POM_LOOP'),'OFF'), + cond.smaller(m.mnemonic('IMIR_HK_ICE_SEC_VOLT1'),1), + cond.greater(m.mnemonic('SE_ZIMIRICEA'),0.2)] + + generate object of condition with the con_set as attribute: + >>>condition_object=cond.condition(con_set) + + Now the condition_object can return a True/False statement wheather + the time given as attribut meets the conditions: + + >>>if condition.state(float(element['Primary Time'])): + -> True when condition for the given time applies + -> False when condition for the given time is not applicable + +Dependencies +------------ + no external files needed + +References +---------- + +Notes +----- + +""" + + +class condition: + """Class to hold several subconditions""" + + #contains a list of all necessary time pairs + cond_time_pairs = [] + #state of the condition + __state = False + + #initializes condition through condition set + def __init__(self, cond_set): + """Initialize object with set of conditions + Parameters + ---------- + cond_set : list + list contains subconditions objects + """ + self.cond_set = cond_set + + #destructor -> take care that all time_pairs are deleted! + def __del__(self): + """Delete object - destructor method""" + del self.cond_time_pairs[:] + + #prints all stored time pairs (for developement only) + def print_times(self): + """Print conditions time pairs on command line (developement)""" + print('Available time pairs:') + for times in self.cond_time_pairs: + print('list: '+str(times)) + + #returns a interval if time is anywhere in between + def get_interval(self, time): + """Returns time interval if availlable, where "time" is in between + Parameters + ---------- + time : float + given time attribute + Return + ------ + time_pair : tuple + pair of start_time and end_time where time is in between + """ + + end_time = 10000000 + start_time = 0 + + #do for every condition + for cond in self.cond_time_pairs: + #do for every time pair in condition + for pair in cond: + if (time > pair[0]) and (time < pair[1]): + if (end_time > pair[1]) and (start_time < pair[0]): + start_time = pair[0] + end_time = pair[1] + break + else: + break + + if (end_time != 10000000) and (start_time != 0): + return (start_time, end_time) + else: + return None + + + #generates time pairs out of start and end times + def generate_time_pairs(start_times, end_times): + """Forms time pairs out of start times and end times + Parameters + ---------- + start_times : list + contains all times where a condition applies + end_times : list + contains all times where the condition does not apply + Return + ------ + time_pair : list + list of touples with start and end time + """ + #internal use only + time_pair: float = [] + + #when the conditons doesn´t apply anyway + if not start_times: + time_pair.append((0,0)) + + #check if the condition indicates an open time range + elif not end_times: + time_pair.append((start_times[0], 0)) + + #generate time pairs + #for each start time a higher or equal end time is searched for + #these times form am touple which is appended to time_pair : list + else: + time_hook = 0 + last_start_time = 0 + + for start in list(sorted(set(start_times))): + + if(start > time_hook): + for end in list(sorted(set(end_times))): + + if end > start: + + time_pair.append((start, end)) + time_hook = end + break + + if list(sorted(set(start_times)))[-1] > list(sorted(set(end_times)))[-1]: + time_pair.append((list(sorted(set(end_times)))[-1], 0)) + + return(time_pair) + + + #returns state of the condition at a given time + #if state(given time)==True -> condition is true + #if state(given time)==False -> condition is false + def state(self, time): + """Checks whether condition is true of false at a given time + Parameters + ---------- + time : float + input time for condition query + Return + ------ + state : bool + True/False statement whether the condition applies or not + """ + #checks condition for every sub condition in condition set (subconditions) + + state = self.__state + + for cond in self.cond_time_pairs: + + if self.__check_subcondition(cond, time): + state = True + else: + state = False + break + + return state + + + def __check_subcondition(self, cond, time): + + #if there are no values availlable + if cond[0][0] == 0: + return False + + for time_pair in cond: + #if just a positive time is availlable, return true + if (time_pair[1] == 0) and (time > time_pair[0]): + + return True + + #if given time occurs between a time pair, return true + elif (time_pair[0]) <= time and (time < time_pair[1]): + + return True + + else: + pass + + +class equal(condition): + """Class to hold single "is equal" subcondition""" + + #add attributes to function - start function "cond_time_pairs()" + def __init__(self, mnemonic, value): + """Initializes subconditon + Parameters + ---------- + mnemonic : astropy table + includes mnemomic engineering data and corresponding primary time + value : str + coparison value for equal statement + """ + self.mnemonic = mnemonic + self.value = value + + condition.cond_time_pairs.append((self.cond_true_time())) + + + #generates a list of time-touples (start_time, end_time) that mark the beginning and end of + #wheather the condition is true or not + def cond_true_time(self): + """Filters all values that are equal to a given comparison value + if equal: Primary time -> temp_start + if not equal: Primary time -> temp_end + Return + ------ + time_p : list + list of touples with start and end time + """ + temp_start = [] + temp_end = [] + + for key in self.mnemonic: + + #find all times whoses Raw values equal the given value + if key['value'] == self.value: + temp_start.append(key["time"]) + + #find all end values + else: + temp_end.append(key["time"]) + + time_p = condition.generate_time_pairs(temp_start, temp_end) + + return time_p + + +class greater(condition): + """Class to hold single "greater than" subcondition""" + + #add attributes to function - start function "cond_time_pairs()" + def __init__(self, mnemonic, value): + """Initializes subconditon + Parameters + ---------- + mnemonic : astropy table + includes mnemomic engineering data and corresponding primary time + value : str + coparison value for equal statement + """ + self.mnemonic=mnemonic + self.value=value + + condition.cond_time_pairs.append((self.cond_true_time())) + + def cond_true_time(self): + """Filters all values that are greater than a given comparison value + if equal: Primary time -> temp_start + if not equal: Primary time -> temp_end + Return + ------ + time_p : list + list of touples with start and end time + """ + temp_start: float = [] + temp_end: float = [] + + for key in self.mnemonic: + + #find all times whose Raw values are grater than the given value + if float(key['value']) > self.value: + temp_start.append(key["time"]) + + #find all end values + else: + temp_end.append(key["time"]) + + return condition.generate_time_pairs(temp_start, temp_end) + +class smaller(condition): + """Class to hold single "greater than" subcondition""" + + #add attributes to function - start function "cond_time_pairs()" + def __init__(self, mnemonic, value): + """Initializes subconditon + Parameters + ---------- + mnemonic : astropy table + includes mnemomic engineering data and corresponding primary time + value : str + coparison value for equal statement + """ + self.mnemonic=mnemonic + self.value=value + + condition.cond_time_pairs.append((self.cond_true_time())) + + def cond_true_time(self): + """Filters all values that are greater than a given comparison value + if equal: Primary time -> temp_start + if not equal: Primary time -> temp_end + Return + ------ + time_p : list + list of touples with start and end time + """ + temp_start: float = [] + temp_end: float = [] + + for key in self.mnemonic: + + #find all times whose Raw values are grater than the given value + if float(key['value']) < self.value: + temp_start.append(key["time"]) + + #find all end values + else: + temp_end.append(key["time"]) + + return condition.generate_time_pairs(temp_start, temp_end) + + +if __name__ =='__main__': + pass + diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/csv_to_AstropyTable.py b/jwql/instrument_monitors/miri_monitors/data_trending/csv_to_AstropyTable.py new file mode 100755 index 000000000..104078839 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/csv_to_AstropyTable.py @@ -0,0 +1,150 @@ +#! /usr/bin/env python +""" + +Module for retrieving and evaluating mnemonic data. +The module contais... + + +Authors +------- + + - Daniel Kühbacher + +Use +--- + + +Dependencies +------------ + + mnemonics.py -> includes a list of mnemonics to be evaluated + +References +---------- + + +Notes +----- + +""" +from astropy.table import Table +from astropy.time import Time +import mnemonics as mn + + +class mnemonics: + """class to hold a set of mnemonics""" + + __mnemonic_dict = {} + __mnemonic_label = "Telemetry Mnemonic" + __time_label = "Secondary Time" + __value_label = "EU Value" + + + def __init__(self, import_path): + """main function of this class + Parameters + ---------- + import_path : str + defines file to import (csv sheet) + """ + + imported_data = self.import_CSV(import_path) + length = len(imported_data) + + print('{} was imported - {} lines'.format(import_path, length)) + + #look for every mnmonic given in mnemonicy.py + for mnemonic_name in mn.mnemonic_set_base: + temp = self.sort_mnemonic(mnemonic_name, imported_data) + #append temp to dict with related mnemonic + self.__mnemonic_dict.update({mnemonic_name:temp}) + + + def import_CSV(self, path): + """imports csv sheet and converts it to AstropyTable + Parameters + ---------- + path : str + defines path to file to import + Return + ------ + imported_data : AstropyTable + container for imported data + """ + + #read data from given *CSV file + imported_data=Table.read(path, format='ascii.basic', delimiter=',') + return imported_data + + + #returns table of single mnemonic + def mnemonic(self, name): + """Returns table of one single mnemonic + Parameters + ---------- + name : str + name of mnemonic + Return + ------ + __mnemonic_dict[name] : AstropyTable + corresponding table to mnemonic name + """ + try: + return self.__mnemonic_dict[name] + except KeyError: + print('{} not in list'.format(name)) + + + #looks for given mnemonic in given table + #returns list containing astropy tables with sorted mnemonics and engineering values + #adds useful meta data to Table + def sort_mnemonic(self, mnemonic, table): + """Looks for all values in table with identifier "mnemonic" + Converts time string to mjd format + Parameters + ---------- + mnemonic : str + identifies which mnemonic to look for + table : AstropyTable + table that stores mnemonics and data + Return + ------ + mnemonic_table : AstropyTable + stores all data associated with identifier "mnemonic" + """ + + temp1: float = [] + temp2 = [] + + #appends present mnemonic data to temp arrays temp1 and temp2 + for item in table: + if item[self.__mnemonic_label] == mnemonic: + #convert time string to mjd format + temp = item[self.__time_label].replace('/','-').replace(' ','T') + t = Time(temp, format='isot') + + temp1.append(t.mjd) + temp2.append(item[self.__value_label]) + + description = ('time','value') + data = [temp1, temp2] + + #add some meta data + if len(temp1) > 0: + date_start = temp1[0] + date_end = temp1[len(temp1)-1] + info = {'start':date_start, 'end':date_end} + else: + info = {"n":"n"} + + #add name of mnemonic to meta data of list + info['mnemonic'] = mnemonic + info['len'] = len(temp1) + + #table to return + mnemonic_table = Table(data, names = description, dtype = ('f8','str'), meta = info) + return mnemonic_table + +if __name__ =='__main__': + pass \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/data_extract.py b/jwql/instrument_monitors/miri_monitors/data_trending/data_extract.py new file mode 100755 index 000000000..444b26e7f --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/data_extract.py @@ -0,0 +1,287 @@ +"""Functions that are used to extract data + + +------- + - Daniel Kühbacher + +Use +--- + + +Dependencies +------------ + no external files needed + +References +---------- + +Notes +----- + +""" + +import mnemonics as mn +import condition as cond +import statistics +import sqlite3 +from collections import defaultdict + + +def extract_data(condition, mnemonic): + '''Function extracts data from given mnemmonic that applies to the given condition + Parameters + ---------- + condition : object + conditon object that holds one or more subconditions + mnemonic : AstropyTable + holds single table with mnemonic data + Return + ------ + temp : list or None + holds data that applies to given condition + ''' + temp = [] + + #look for all values that fit to the given conditions + for element in mnemonic: + if condition.state(float(element['time'])): + #developement purpose: + #print("condition true: value= {}, time= {}".format(element['value'],element['time'])) + temp.append(float(element['value'])) + + if len(temp) > 0: + return temp + else: + return None + +def extract_filterpos(condition, nominals, ratio_mnem, pos_mnem): + '''Extracts to filterpositions corresponding ratio values + Parameters + ---------- + condition : object + conditon object that holds one or more subconditions + ratio_mem : AstropyTable + holds ratio values of one specific mnemonic + pos_mem : AstropyTable + holds pos values of one specific mnemonic + Return + ------ + pos_values : dict + holds ratio values and times with corresponding pos label + ''' + + pos_values = defaultdict(list) + + #do for every position in mnemonic + for pos in pos_mnem: + + interval = condition.get_interval(pos['time']) + + if interval is not None: + cur_pos_time = pos['time'] + filtername = pos['value'] + + for ratio in ratio_mnem: + + if (ratio['time'] >= cur_pos_time) and \ + (abs(float(ratio['value']) - nominals.get(pos['value'])) < 10): + + if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): + pos_values[pos['value']].append(( ratio['time'], ratio['value'])) + break + + return pos_values + + +def once_a_day_routine(mnemonic_data): + + #abbreviate attribute + m = mnemonic_data + + ######################################################################### + con_set_1 = [ \ + cond.equal(m.mnemonic('IMIR_HK_IMG_CAL_LOOP'),'OFF'), \ + cond.equal(m.mnemonic('IMIR_HK_IFU_CAL_LOOP'),'OFF'), \ + cond.equal(m.mnemonic('IMIR_HK_POM_LOOP'),'OFF'), \ + cond.smaller(m.mnemonic('IMIR_HK_ICE_SEC_VOLT1'),1.0), \ + cond.greater(m.mnemonic('SE_ZIMIRICEA'),0.2)] + #setup condition + condition_1 = cond.condition(con_set_1) + + data_cond_1 = dict() + + for identifier in mn.mnemonic_cond_1: + data = extract_data(condition_1, m.mnemonic(identifier)) + + if data != None: + data_cond_1.update( {identifier:data} ) + else: + print("no data for {}".format(identifier)) + + del condition_1 + + ########################################################################## + con_set_2 = [ \ + cond.greater(m.mnemonic('SE_ZIMIRFPEA'), 0.5), \ + cond.equal(m.mnemonic('IGDP_IT_MIR_IC_STATUS'), 'DETECTOR_READY'), \ + cond.equal(m.mnemonic('IGDP_IT_MIR_LW_STATUS'), 'DETECTOR_READY')] + #setup condition + condition_2 = cond.condition(con_set_2) + + + data_cond_2 = dict() + + for identifier in mn.mnemonic_cond_2: + data = extract_data(condition_2, m.mnemonic(identifier)) + + if data != None: + data_cond_2.update( {identifier:data} ) + else: + print("no data for {}".format(identifier)) + + del condition_2 + + return data_cond_1, data_cond_2 + +#in developement +def whole_day_routine(mnemonic_data): + + #abbreviate attribute + m = mnemonic_data + + ######################################################################### + con_set_3 = [ \ + cond.greater(m.mnemonic('IMIR_HK_ICE_SEC_VOLT1'), 25.0)] + #setup condition + condition_3 = cond.condition(con_set_3) + + data_cond_3 = dict() + + for identifier in mn.mnemonic_cond_3: + data = extract_data(condition_3, m.mnemonic(identifier)) + + if data != None: + data_cond_3.update({identifier:data}) + else: + print("no data for {}".format(identifier)) + + del condition_3 + + ######################################################################### + con_set_FW = [ \ + cond.greater(m.mnemonic('IMIR_HK_FW_POS_VOLT'),250.0)] + #setup condition + condition_FW = cond.condition(con_set_FW) + FW_volt = extract_data(condition_FW, m.mnemonic('IMIR_HK_FW_POS_VOLT')) + + del condition_FW + + con_set_GW14 = [ \ + cond.greater(m.mnemonic('IMIR_HK_GW14_POS_VOLT'),250.0)] + #setup condition + condition_GW14 = cond.condition(con_set_GW14) + GW14_volt = extract_data(condition_GW14, m.mnemonic('IMIR_HK_GW14_POS_VOLT')) + + del condition_GW14 + + con_set_GW23 = [ \ + cond.greater(m.mnemonic('IMIR_HK_GW23_POS_VOLT'),250.0)] + #setup condition + condition_GW23 = cond.condition(con_set_GW23) + GW23_volt = extract_data(condition_GW23, m.mnemonic('IMIR_HK_GW23_POS_VOLT')) + + del condition_GW23 + + con_set_CCC = [ \ + cond.greater(m.mnemonic('IMIR_HK_CCC_POS_VOLT'),250.0)] + #setup condition + condition_CCC = cond.condition(con_set_CCC) + CCC_volt = extract_data(condition_CCC, m.mnemonic('IMIR_HK_CCC_POS_VOLT')) + + del condition_CCC + + return data_cond_3, FW_volt , GW14_volt, GW23_volt, CCC_volt + + + +def wheelpos_routine(mnemonic_data): + + #abbreviate attribute + m = mnemonic_data + + con_set_FW = [ \ + cond.greater(m.mnemonic('IMIR_HK_FW_POS_VOLT'),250.0)] + #setup condition + condition_FW = cond.condition(con_set_FW) + FW = extract_filterpos(condition_FW, mn.fw_nominals, \ + m.mnemonic('IMIR_HK_FW_POS_RATIO'), m.mnemonic('IMIR_HK_FW_CUR_POS')) + + del condition_FW + + con_set_GW14 = [ \ + cond.greater(m.mnemonic('IMIR_HK_GW14_POS_VOLT'),250.0)] + #setup condition + condition_GW14 = cond.condition(con_set_GW14) + GW14 = extract_filterpos(condition_GW14, mn.gw14_nominals, \ + m.mnemonic('IMIR_HK_GW14_POS_RATIO'), m.mnemonic('IMIR_HK_GW14_CUR_POS')) + + del condition_GW14 + + con_set_GW23 = [ \ + cond.greater(m.mnemonic('IMIR_HK_FW_POS_VOLT'),250.0)] + #setup condition + condition_GW23 = cond.condition(con_set_FW) + GW23 = extract_filterpos(condition_GW23, mn.gw23_nominals, \ + m.mnemonic('IMIR_HK_GW23_POS_RATIO'), m.mnemonic('IMIR_HK_GW23_CUR_POS')) + + del condition_GW23 + + con_set_CCC = [ \ + cond.greater(m.mnemonic('IMIR_HK_CCC_POS_VOLT'),250.0)] + #setup condition + condition_CCC = cond.condition(con_set_FW) + CCC = extract_filterpos(condition_CCC, mn.fw_nominals, \ + m.mnemonic('IMIR_HK_CCC_POS_RATIO'), m.mnemonic('IMIR_HK_CCC_CUR_POS')) + + del condition_CCC + + return FW, GW14, GW23, CCC + +def extract_filterpos_obsolete(condition, ratio_mnem, pos_mnem): + '''Extracts to filterpositions corresponding ratio values + Parameters + ---------- + condition : object + conditon object that holds one or more subconditions + ratio_mem : AstropyTable + holds ratio values of one specific mnemonic + pos_mem : AstropyTable + holds pos values of one specific mnemonic + Return + ------ + pos_values : dict + holds ratio values and times with corresponding pos label + ''' + + pos_values = defaultdict(list) + + #do for every position in mnemonic + for pos in pos_mnem: + + interval = condition.get_interval(pos['time']) + + if interval is not None: + cur_pos_time = pos['time'] + filtername = pos['value'] + + for ratio in ratio_mnem: + + if (ratio['time'] == cur_pos_time) and (abs(float(ratio['value'])) > 0.1): + if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): + pos_values[pos['value']].append( ( ratio['time'], ratio['value']) ) + break + return pos_values + + +if __name__ =='__main__': + pass diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py b/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py new file mode 100755 index 000000000..6fa29e7b3 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py @@ -0,0 +1,154 @@ + +import statistics +import sqlite3 + +import mnemonics as mn +import sql_interface as sql +import condition as cond + +import csv_to_AstropyTable as apt + +from data_extract import whole_day_routine, wheelpos_routine + + + +#create filename string +directory='/home/daniel/STScI/trainigData/set_1_day/' +""" +filenames = [ +'imir_190204_DoY2292017FOFTLM2019035182609402.CSV', +'imir_190204_DoY2402017FOFTLM2019035180907145.CSV', +'imir_190204_DoY2412017FOFTLM2019035181004311.CSV', +'imir_190204_DoY2312017FOFTLM2019035184159965.CSV', +'imir_190204_DoY2422017FOFTLM2019035181027504.CSV', +'imir_190204_DoY2322017FOFTLM2019035184236985.CSV', +""" +filenames1 = [ +'imir_190204_DoY2432017FOFTLM2019035181100606.CSV', +'imir_190204_DoY2332017FOFTLM2019035184306076.CSV', +'imir_190204_DoY2442017FOFTLM2019035181126853.CSV', +'imir_190204_DoY2342017FOFTLM2019035184347174.CSV', +'imir_190204_DoY2452017FOFTLM2019035181155234.CSV', +'imir_190204_DoY2352017FOFTLM2019035184708935.CSV', +'imir_190204_DoY2462017FOFTLM2019035181230871.CSV', +'imir_190204_DoY2362017FOFTLM2019035184737246.CSV', +'imir_190204_DoY2472017FOFTLM2019035181252890.CSV', +'imir_190204_DoY2372017FOFTLM2019035184806338.CSV', +'imir_190204_DoY2482017FOFTLM2019035181322838.CSV', +'imir_190204_DoY2382017FOFTLM2019035184832737.CSV', +'imir_190204_DoY2492017FOFTLM2019035181406861.CSV', +'imir_190204_DoY2392017FOFTLM2019035180833486.CSV', +'imir_190204_DoY2502017FOFTLM2019035181519288.CSV'] + +filenames = ['imir_190204_DoY2302017FOFTLM2019035184123736.CSV'] + +def process_file(conn, path): + m_raw_data = apt.mnemonics(path) + + cond3, FW_volt, GW14_volt, GW23_volt, CCC_volt = whole_day_routine(m_raw_data) + FW, GW14, GW23, CCC= wheelpos_routine(m_raw_data) + + #put data from con3 to database + for key, value in cond3.items(): + + m = m_raw_data.mnemonic(key) + + if value != None: + if len(value) > 2: + if key == "SE_ZIMIRICEA": + length = len(value) + mean = statistics.mean(value) + deviation = statistics.stdev(value) + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, "SE_ZIMIRICEA_HV_ON", dataset) + + elif key == "IMIR_HK_ICE_SEC_VOLT4": + length = len(value) + mean = statistics.mean(value) + deviation = statistics.stdev(value) + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, "IMIR_HK_ICE_SEC_VOLT4_HV_ON", dataset) + + else: + length = len(value) + mean = statistics.mean(value) + deviation = statistics.stdev(value) + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, key, dataset) + ######################################################################################### + if FW_volt != None: + if len(FW_volt) > 2: + length = len(FW_volt) + mean = statistics.mean(FW_volt) + deviation = statistics.stdev(FW_volt) + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, "IMIR_HK_FW_POS_VOLT", dataset) + + if GW14_volt != None: + if len(GW14_volt) > 2: + length = len(GW14_volt) + mean = statistics.mean(GW14_volt) + deviation = statistics.stdev(GW14_volt) + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, "IMIR_HK_GW14_POS_VOLT", dataset) + + if GW23_volt != None: + if len(GW23_volt) > 2: + length = len(GW23_volt) + mean = statistics.mean(GW23_volt) + deviation = statistics.stdev(GW23_volt) + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, "IMIR_HK_GW23_POS_VOLT", dataset) + + if CCC_volt != None: + if len(CCC_volt) > 2: + length = len(CCC_volt) + mean = statistics.mean(CCC_volt) + deviation = statistics.stdev(CCC_volt) + dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) + sql.add_data(conn, "IMIR_HK_CCC_POS_VOLT", dataset) + + ######################################################################################### + for pos in mn.fw_positions: + try: + data = FW[pos] + for element in data: + sql.add_wheel_data(conn, 'IMIR_HK_FW_POS_RATIO_{}'.format(pos), element) + except KeyError: + pass + + for pos in mn.gw_positions: + try: + data_GW14 = GW14[pos] + data_GW23 = GW14[pos] + + for element in data_GW14: + sql.add_wheel_data(conn, 'IMIR_HK_GW14_POS_RATIO_{}'.format(pos), element) + for element in data_GW23: + sql.add_wheel_data(conn, 'IMIR_HK_GW23_POS_RATIO_{}'.format(pos), element) + except KeyError: + pass + + for pos in mn.ccc_positions: + try: + data = CCC[pos] + for element in data: + sql.add_wheel_data(conn, 'IMIR_HK_CCC_POS_RATIO_{}'.format(pos), element) + except KeyError: + pass + + + +def main(): + db_file = "miri_database.db" + conn = sql.create_connection(db_file) + + for name in filenames: + path = directory + name + process_file(conn, path) + + sql.close_connection(conn) + print("done") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/filterpos_extract.py b/jwql/instrument_monitors/miri_monitors/data_trending/filterpos_extract.py new file mode 100755 index 000000000..d2558ad65 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/filterpos_extract.py @@ -0,0 +1,112 @@ + + +import csv_to_AstropyTable as apt +import statistics +import sqlite3 + +import mnemonics as mn +import sql_interface as sql +import condition as cond + +from collections import defaultdict + + + +#create filename string +directory='/home/daniel/STScI/trainigData/set_1_day/' + +filename= 'imir_190204_DoY2292017FOFTLM2019035182609402.CSV' + +filterpositions = [ +'FND', +'OPAQUE', +'F1000W', +'F1130W', +'F1280W', +'P750L', +'F1500W', +'F1800W', +'F2100W', +'F560W', +'FLENS', +'F2300C', +'F770W', +'F1550C', +'F2550W', +'F1140C', +'F2550WR', +'F1065C'] + +def extract_filterpos(cond, ratio_mnem, pos_mnem): + + pos_values = defaultdict(list) + + #do for every position in file + for pos in pos_mnem: + + interval = cond.get_interval(pos['time']) + + if interval is not None: + cur_pos_time = pos['time'] + filtername = pos['value'] + + for ratio in ratio_mnem: + + if (ratio['time'] > cur_pos_time) and (abs(float(ratio['value'])) > 0.1): + if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): + pos_values[pos['value']].append( ( ratio['time'], ratio['value']) ) + break + return pos_values + + + +def day_routine(path): + + #read data in file "path" and return dictionary with mnemonic and astropy table + m = apt.mnemonics(path) + + + #prepare condition for data retrieval + con_set_3 = [ + cond.greater(m.mnemonic('IMIR_HK_FW_POS_VOLT'), 250.0)] + #setup condition + condition_3=cond.condition(con_set_3) + + v = extract_filterpos(condition_3, m.mnemonic('IMIR_HK_FW_POS_RATIO'), m.mnemonic('IMIR_HK_FW_CUR_POS')) + print (v) + + + +# create initial database or check if all tables are setup + +path = directory+filename +day_routine(path) + + +print('programm end') +##################################### +#programm end + +""" + for element in m.mnemonic('IMIR_HK_FW_CUR_POS'): + + if condition_3.state(element['Secondary Time']): + cur_pos_time = element['Secondary Time'] + interval = condition_3.get_interval(element['Secondary Time'],0) + filtername = element['EU Value'] + + if interval is not None: + for item in m.mnemonic('IMIR_HK_FW_POS_RATIO'): + if (item['Secondary Time'] > cur_pos_time) and (abs(float(item['EU Value'])) > 0.1): + if (item['Secondary Time'] > interval[0]) and (item['Secondary Time'] < interval[1]): + + pos_values[] + print(filtername+'={}'.format(item['EU Value'])) + break + else: + pass + else: + pass + else: + pass +""" \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/main.py b/jwql/instrument_monitors/miri_monitors/data_trending/main.py new file mode 100755 index 000000000..187631bcd --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/main.py @@ -0,0 +1,143 @@ +import numpy as np +import astropy as ap +import functions as func +import sql_interface as sql +import statistics + +from bokeh.plotting import figure, output_file, show + + +testdata=[ +'imir_190130_otis229FOFTLM2019030204146194.CSV', +'imir_190130_otis240FOFTLM2019030210631185.CSV', +'imir_190130_otis230FOFTLM2019030204240886.CSV', +'imir_190130_otis241FOFTLM2019030210651672.CSV', +'imir_190130_otis231FOFTLM2019030204334644.CSV', +'imir_190130_otis242FOFTLM2019030210728909.CSV', +'imir_190130_otis232FOFTLM2019030204455835.CSV', +'imir_190130_otis243FOFTLM2019030210744062.CSV', +'imir_190130_otis233FOFTLM2019030204521412.CSV', +'imir_190130_otis244FOFTLM2019030210809362.CSV', +'imir_190130_otis234FOFTLM2019030204555665.CSV', +'imir_190130_otis245FOFTLM2019030210828095.CSV', +'imir_190130_otis235FOFTLM2019030204617145.CSV', +'imir_190130_otis246FOFTLM2019030210852965.CSV', +'imir_190130_otis236FOFTLM2019030204651604.CSV', +'imir_190130_otis247FOFTLM2019030210914141.CSV', +'imir_190130_otis237FOFTLM2019030204712019.CSV', +'imir_190130_otis248FOFTLM2019030210940944.CSV', +'imir_190130_otis238FOFTLM2019030204738855.CSV', +'imir_190130_otis249FOFTLM2019030211002524.CSV', +'imir_190130_otis239FOFTLM2019030204805611.CSV', +'imir_190130_otis250FOFTLM2019030211032094.CSV'] + +#programm start +################################################## + +#default path adress for trainigData + +directory='/home/daniel/STScI/trainigData/' + + + +################################################## + +""" +Condition 1 for mnemonics below: + +HV not active. +IMIR_HK_ICE_SEC_VOLT1<1V + +Cal sources inactive +IMIR_HK_IMG_CAL_LOOP=OFF +IMIR_HK_IFU_CAL_LOOP=OFF + +POM heater inactive +IMIR_HK_POM_LOOP=OFF + +###################################### +SE_ZIMIRICEA +GP_ZPSVOLT +IMIR_HK_ICE_SEC_VOLT4 +IGDP_MIR_ICE_INTER_TEMP +SI_GZMPT1AK +IGDP_MIR_ICE_T1P_CRYO +IGDP_MIR_ICE_T2R_CRYO +IGDP_MIR_ICE_T3LW_CRYO +IGDP_MIR_ICE_T4SW_CRYO +IGDP_MIR_ICE_T5IMG_CRYO +IGDP_MIR_ICE_T6DECKCRYO +IGDP_MIR_ICE_T7IOC_CRYO +IGDP_MIR_ICE_FW_AMB +IGDP_MIR_ICE_CCC_CRYO +IGDP_MIR_ICE_GW14_CRYO +IGDP_MIR_ICE_GW23_CRYO +IGDP_MIR_ICE_POMP_CRYO +IGDP_MIR_ICE_POMR_CRYO +IGDP_MIR_ICE_IFU_CRYO +IGDP_MIR_ICE_IMG_CRYO +###################################### +""" + + +output=[] + + + +path = directory+test_data[1] + + +i=imported_file=func.mnemonics(path) + +#setup condition for data retrieval +con_set_1 = [ \ +func.equal(i.mnemonic('IMIR_HK_IMG_CAL_LOOP'),'OFF'), \ +func.equal(i.mnemonic('IMIR_HK_IFU_CAL_LOOP'),'OFF'), \ +func.equal(i.mnemonic('IMIR_HK_POM_LOOP'),'OFF'), \ +func.smaller(i.mnemonic('IMIR_HK_ICE_SEC_VOLT1'),1) ] + +condition_1=func.condition(con_set_1) + +useful_data=[] + +for element in i.mnemonic('IMIR_HK_ICE_SEC_VOLT4'): + if condition_1.state(element['Primary Time']): + useful_data.append(float(element['EU Value'])) + +try: + average=sum(useful_data)/len(useful_data) + deviation=statistics.stdev(useful_data) + + output.append((average,deviation)) + print("Average of SE_ZIMIRICEA ({} values) is: {} , Deviation: {}" \ + .format(len(useful_data),average, deviation)) + +except ZeroDivisionError: + print('No useful data for SE_ZIMIRICEA') + + +print(output) + + + + +#simple bokeh plot for retrieved and analysed data +x=np.linspace(0,len(output)) +y1=[] + +for element in output: + y1.append(element[0]) + + +output_file("line.html") + +p = figure(plot_width=400, plot_height=400) + +# add a line renderer +p.line(x,y1, line_width=2) + +show(p) + + +################################################## +#programm end diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/mnemonics.py b/jwql/instrument_monitors/miri_monitors/data_trending/mnemonics.py new file mode 100755 index 000000000..18417cac9 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/mnemonics.py @@ -0,0 +1,428 @@ +"""Module lists all neccessary mnemonics for MIRI data trending + +The module includes several lists to import to MIRI data trending monitor program. +The lists are used for data aquisation and to set up the initial database. + +Authors +------- + - Daniel Kühbacher + +Use +--- +prefered import: + import 00_mnemoncis as mn + +Dependencies +------------ + further information to included mnemonics: ############### + +References +---------- + +Notes +----- +The lists are for developement purpose only. + +""" + +#all mnemonic used for condition 1 (see: draft) +#"SE_ZBUSVLT", +mnemonic_cond_1 = [ +"SE_ZIMIRICEA", + +"IMIR_HK_ICE_SEC_VOLT4", +"IGDP_MIR_ICE_INTER_TEMP", + +"ST_ZTC1MIRIA", +"ST_ZTC1MIRIB", + +"IGDP_MIR_ICE_T1P_CRYO", +"IGDP_MIR_ICE_T2R_CRYO", +"IGDP_MIR_ICE_T3LW_CRYO", +"IGDP_MIR_ICE_T4SW_CRYO", +"IGDP_MIR_ICE_T5IMG_CRYO", +"IGDP_MIR_ICE_T6DECKCRYO", +"IGDP_MIR_ICE_T7IOC_CRYO", +"IGDP_MIR_ICE_FW_CRYO", +"IGDP_MIR_ICE_CCC_CRYO", +"IGDP_MIR_ICE_GW14_CRYO", +"IGDP_MIR_ICE_GW23_CRYO", +"IGDP_MIR_ICE_POMP_CRYO", +"IGDP_MIR_ICE_POMR_CRYO", +"IGDP_MIR_ICE_IFU_CRYO", +"IGDP_MIR_ICE_IMG_CRYO"] + +#all mnemonics used for condition 2 (see: draft) +mnemonic_cond_2=[ +"SE_ZIMIRFPEA", + +"IMIR_PDU_V_DIG_5V", +"IMIR_PDU_I_DIG_5V", +"IMIR_PDU_V_ANA_5V", +"IMIR_PDU_I_ANA_5V", + +"IMIR_PDU_V_ANA_N5V", +"IMIR_PDU_I_ANA_N5V", + +"IMIR_PDU_V_ANA_7V", +"IMIR_PDU_I_ANA_7V", + +"IMIR_PDU_V_ANA_N7V", +"IMIR_PDU_I_ANA_N7V", + +"IMIR_SPW_V_DIG_2R5V", +"IMIR_PDU_V_REF_2R5V", + +"IGDP_MIR_IC_V_VDETCOM", +"IGDP_MIR_SW_V_VDETCOM", +"IGDP_MIR_LW_V_VDETCOM", + +"IGDP_MIR_IC_V_VSSOUT", +"IGDP_MIR_SW_V_VSSOUT", +"IGDP_MIR_LW_V_VSSOUT", +"IGDP_MIR_IC_V_VRSTOFF", + +"IGDP_MIR_SW_V_VRSTOFF", +"IGDP_MIR_LW_V_VRSTOFF", + +"IGDP_MIR_IC_V_VP", +"IGDP_MIR_SW_V_VP", +"IGDP_MIR_LW_V_VP", + +"IGDP_MIR_IC_V_VDDUC", +"IGDP_MIR_SW_V_VDDUC", +"IGDP_MIR_LW_V_VDDUC", + +"IMIR_PDU_TEMP", + +"ST_ZTC2MIRIA", +"ST_ZTC2MIRIB", + + +"IMIR_IC_SCE_ANA_TEMP1", +"IMIR_SW_SCE_ANA_TEMP1", +"IMIR_LW_SCE_ANA_TEMP1", + +"IMIR_IC_SCE_DIG_TEMP", +"IMIR_SW_SCE_DIG_TEMP", +"IMIR_LW_SCE_DIG_TEMP", + +"IGDP_MIR_IC_DET_TEMP", +"IGDP_MIR_LW_DET_TEMP", +"IGDP_MIR_SW_DET_TEMP"] + +#ICE secondary voltages -> apply to condition3 +mnemonic_cond_3 = [ +"IMIR_HK_ICE_SEC_VOLT1", +"IMIR_HK_ICE_SEC_VOLT2", +"IMIR_HK_ICE_SEC_VOLT3", +"IMIR_HK_ICE_SEC_VOLT4", + +"SE_ZIMIRICEA"] + +#filter weel positions +fw_positions = [ +"FND", +"OPAQUE", +"F1000W", +"F1130W", +"F1280W", +"P750L", +"F1500W", +"F1800W", +"F2100W", +"F560W", +"FLENS", +"F2300C", +"F770W", +"F1550C", +"F2550W", +"F1140C", +"F2550WR", +"F1065C"] + +#grating weel positions +gw_positions = [ +"SHORT", +"MEDUIM", +"LONG"] + +#contamination control clap positions +ccc_positions = [ +"LOCKED", +"OPEN", +"CLOSED"] + + +fw_nominals = { +"FND" : -164.46, +"OPAQUE" : 380.42, +"F1000W" : -23.88, +"F1130W" : 138.04, +"F1280W" : -298.14, +"P750L" : 12.79, +"F1500W" : -377.32, +"F1800W" : 435.61, +"F2100W" : -126.04, +"F560W" : 218.13, +"FLENS" : -212.44, +"F2300C" : 306.03, +"F770W" : -61.90, +"F1550C" : 188.88, +"F2550W" : -323.65, +"F1140C" : 83.08, +"F2550WR" : -255.18, +"F1065C" : 261.62 } + +gw23_nominals = { +"SHORT" : 619.81, +"MEDIUM" : 373.31, +"LONG" : 441.4} + +gw14_nominals = { +"SHORT" : 627.49, +"MEDIUM" : 342.71, +"LONG" : 408.75 } + +ccc_nominals = { +"LOCKED" : 577.23, +"OPEN" : 507.86, +"CLOSED" : 399.90 } + + +#mnemonic set for basic query +#"SE_ZBUSVLT", +mnemonic_set_base = [ +"SE_ZIMIRICEA", + +"IMIR_HK_ICE_SEC_VOLT1", +"IMIR_HK_ICE_SEC_VOLT2", +"IMIR_HK_ICE_SEC_VOLT3", +"IMIR_HK_ICE_SEC_VOLT4", + +"IGDP_MIR_ICE_INTER_TEMP", + +"ST_ZTC1MIRIB", +"ST_ZTC1MIRIA", +"ST_ZTC2MIRIB", +"ST_ZTC2MIRIA", + +"IGDP_MIR_ICE_T1P_CRYO", +"IGDP_MIR_ICE_T2R_CRYO", +"IGDP_MIR_ICE_T3LW_CRYO", +"IGDP_MIR_ICE_T4SW_CRYO", +"IGDP_MIR_ICE_T5IMG_CRYO", +"IGDP_MIR_ICE_T6DECKCRYO", +"IGDP_MIR_ICE_T7IOC_CRYO", +"IGDP_MIR_ICE_FW_CRYO", +"IGDP_MIR_ICE_CCC_CRYO", +"IGDP_MIR_ICE_GW14_CRYO", +"IGDP_MIR_ICE_GW23_CRYO", +"IGDP_MIR_ICE_POMP_CRYO", +"IGDP_MIR_ICE_POMR_CRYO", +"IGDP_MIR_ICE_IFU_CRYO", +"IGDP_MIR_ICE_IMG_CRYO", + +"SE_ZIMIRFPEA", + +"IMIR_PDU_V_DIG_5V", +"IMIR_PDU_I_DIG_5V", +"IMIR_PDU_V_ANA_5V", +"IMIR_PDU_I_ANA_5V", + +"IMIR_PDU_V_ANA_N5V", +"IMIR_PDU_I_ANA_N5V", + +"IMIR_PDU_V_ANA_7V", +"IMIR_PDU_I_ANA_7V", + +"IMIR_PDU_V_ANA_N7V", +"IMIR_PDU_I_ANA_N7V", + +"IMIR_SPW_V_DIG_2R5V", +"IMIR_PDU_V_REF_2R5V", + +"IGDP_MIR_IC_V_VDETCOM", +"IGDP_MIR_SW_V_VDETCOM", +"IGDP_MIR_LW_V_VDETCOM", + +"IGDP_MIR_IC_V_VSSOUT", +"IGDP_MIR_SW_V_VSSOUT", +"IGDP_MIR_LW_V_VSSOUT", +"IGDP_MIR_IC_V_VRSTOFF", + +"IGDP_MIR_SW_V_VRSTOFF", +"IGDP_MIR_LW_V_VRSTOFF", + +"IGDP_MIR_IC_V_VP", +"IGDP_MIR_SW_V_VP", +"IGDP_MIR_LW_V_VP", + +"IGDP_MIR_IC_V_VDDUC", +"IGDP_MIR_SW_V_VDDUC", +"IGDP_MIR_LW_V_VDDUC", + +"IMIR_PDU_TEMP", + +"IMIR_IC_SCE_ANA_TEMP1", +"IMIR_SW_SCE_ANA_TEMP1", +"IMIR_LW_SCE_ANA_TEMP1", + +"IMIR_IC_SCE_DIG_TEMP", +"IMIR_SW_SCE_DIG_TEMP", +"IMIR_LW_SCE_DIG_TEMP", + +"IGDP_MIR_IC_DET_TEMP", +"IGDP_MIR_LW_DET_TEMP", +"IGDP_MIR_SW_DET_TEMP", + +"IMIR_HK_IMG_CAL_LOOP", +"IMIR_HK_IFU_CAL_LOOP", +"IMIR_HK_POM_LOOP", +"IGDP_IT_MIR_IC_STATUS", +"IGDP_IT_MIR_LW_STATUS", +"IGDP_IT_MIR_SW_STATUS", + +"IMIR_HK_FW_POS_VOLT", +"IMIR_HK_FW_POS_RATIO", +"IMIR_HK_FW_CUR_POS", + +"IMIR_HK_GW14_POS_VOLT", +"IMIR_HK_GW14_POS_RATIO", +"IMIR_HK_GW14_CUR_POS", + +"IMIR_HK_GW23_POS_VOLT", +"IMIR_HK_GW23_POS_RATIO", +"IMIR_HK_GW23_CUR_POS", + +"IMIR_HK_CCC_POS_RATIO", +"IMIR_HK_CCC_CUR_POS", +"IMIR_HK_CCC_POS_VOLT" ] + +#mnemonic set for setting up databank +mnemonic_set_database = [ +"SE_ZIMIRICEA_IDLE", +"SE_ZIMIRICEA_HV_ON", + +"SE_ZBUSVLT", + +"IMIR_HK_ICE_SEC_VOLT1", +"IMIR_HK_ICE_SEC_VOLT2", +"IMIR_HK_ICE_SEC_VOLT3", +"IMIR_HK_ICE_SEC_VOLT4_IDLE", +"IMIR_HK_ICE_SEC_VOLT4_HV_ON", + +"IGDP_MIR_ICE_INTER_TEMP", + +"ST_ZTC1MIRIB", +"ST_ZTC1MIRIA", +"ST_ZTC2MIRIB", +"ST_ZTC2MIRIA", + +"IGDP_MIR_ICE_T1P_CRYO", +"IGDP_MIR_ICE_T2R_CRYO", +"IGDP_MIR_ICE_T3LW_CRYO", +"IGDP_MIR_ICE_T4SW_CRYO", +"IGDP_MIR_ICE_T5IMG_CRYO", +"IGDP_MIR_ICE_T6DECKCRYO", +"IGDP_MIR_ICE_T7IOC_CRYO", +"IGDP_MIR_ICE_FW_CRYO", +"IGDP_MIR_ICE_CCC_CRYO", +"IGDP_MIR_ICE_GW14_CRYO", +"IGDP_MIR_ICE_GW23_CRYO", +"IGDP_MIR_ICE_POMP_CRYO", +"IGDP_MIR_ICE_POMR_CRYO", +"IGDP_MIR_ICE_IFU_CRYO", +"IGDP_MIR_ICE_IMG_CRYO", + +"SE_ZIMIRFPEA", + +"IMIR_PDU_V_DIG_5V", +"IMIR_PDU_I_DIG_5V", +"IMIR_PDU_V_ANA_5V", +"IMIR_PDU_I_ANA_5V", + +"IMIR_PDU_V_ANA_N5V", +"IMIR_PDU_I_ANA_N5V", + +"IMIR_PDU_V_ANA_7V", +"IMIR_PDU_I_ANA_7V", + +"IMIR_PDU_V_ANA_N7V", +"IMIR_PDU_I_ANA_N7V", + +"IMIR_SPW_V_DIG_2R5V", +"IMIR_PDU_V_REF_2R5V", + +"IGDP_MIR_IC_V_VDETCOM", +"IGDP_MIR_SW_V_VDETCOM", +"IGDP_MIR_LW_V_VDETCOM", + +"IGDP_MIR_IC_V_VSSOUT", +"IGDP_MIR_SW_V_VSSOUT", +"IGDP_MIR_LW_V_VSSOUT", +"IGDP_MIR_IC_V_VRSTOFF", + +"IGDP_MIR_SW_V_VRSTOFF", +"IGDP_MIR_LW_V_VRSTOFF", + +"IGDP_MIR_IC_V_VP", +"IGDP_MIR_SW_V_VP", +"IGDP_MIR_LW_V_VP", + +"IGDP_MIR_IC_V_VDDUC", +"IGDP_MIR_SW_V_VDDUC", +"IGDP_MIR_LW_V_VDDUC", + +"IMIR_PDU_TEMP", + +"IMIR_IC_SCE_ANA_TEMP1", +"IMIR_SW_SCE_ANA_TEMP1", +"IMIR_LW_SCE_ANA_TEMP1", + +"IMIR_IC_SCE_DIG_TEMP", +"IMIR_SW_SCE_DIG_TEMP", +"IMIR_LW_SCE_DIG_TEMP", + +"IGDP_MIR_IC_DET_TEMP", +"IGDP_MIR_LW_DET_TEMP", +"IGDP_MIR_SW_DET_TEMP", + +"IMIR_HK_FW_POS_VOLT", +"IMIR_HK_GW14_POS_VOLT", +"IMIR_HK_GW23_POS_VOLT", +"IMIR_HK_CCC_POS_VOLT"] + +#different tables for wheelpostions +mnemonic_wheelpositions = [ +"IMIR_HK_FW_POS_RATIO_FND", +"IMIR_HK_FW_POS_RATIO_OPAQUE", +"IMIR_HK_FW_POS_RATIO_F1000W", +"IMIR_HK_FW_POS_RATIO_F1130W", +"IMIR_HK_FW_POS_RATIO_F1280W", +"IMIR_HK_FW_POS_RATIO_P750L", +"IMIR_HK_FW_POS_RATIO_F1500W", +"IMIR_HK_FW_POS_RATIO_F1800W", +"IMIR_HK_FW_POS_RATIO_F2100W", +"IMIR_HK_FW_POS_RATIO_F560W", +"IMIR_HK_FW_POS_RATIO_FLENS", +"IMIR_HK_FW_POS_RATIO_F2300C", +"IMIR_HK_FW_POS_RATIO_F770W", +"IMIR_HK_FW_POS_RATIO_F1550C", +"IMIR_HK_FW_POS_RATIO_F2550W", +"IMIR_HK_FW_POS_RATIO_F1140C", +"IMIR_HK_FW_POS_RATIO_F2550WR", +"IMIR_HK_FW_POS_RATIO_F1065C", + +"IMIR_HK_GW14_POS_RATIO_SHORT", +"IMIR_HK_GW14_POS_RATIO_MEDIUM", +"IMIR_HK_GW14_POS_RATIO_LONG", + + +"IMIR_HK_GW23_POS_RATIO_SHORT", +"IMIR_HK_GW23_POS_RATIO_MEDIUM", +"IMIR_HK_GW23_POS_RATIO_LONG", + +"IMIR_HK_CCC_POS_RATIO_LOCKED", +"IMIR_HK_CCC_POS_RATIO_OPEN", +"IMIR_HK_CCC_POS_RATIO_CLOSED"] diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/plotting.py b/jwql/instrument_monitors/miri_monitors/data_trending/plotting.py new file mode 100755 index 000000000..ff418f846 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/plotting.py @@ -0,0 +1,92 @@ +import sql_interface as sql +import mnemonics as mn + +from os.path import join, dirname +import datetime + +import pandas as pd +from scipy.signal import savgol_filter + +from bokeh.io import curdoc +from bokeh.layouts import row, column +from bokeh.models import ColumnDataSource, DataRange1d, Select +from bokeh.palettes import Blues4 +from bokeh.plotting import figure + +STATISTICS = ['record_min_temp', 'actual_min_temp', 'average_min_temp', 'average_max_temp', 'actual_max_temp', 'record_max_temp'] + +def get_dataset(src, name, distribution): + df = src[src.airport == name].copy() + del df['airport'] + df['date'] = pd.to_datetime(df.date) + # timedelta here instead of pd.DateOffset to avoid pandas bug < 0.18 (Pandas issue #11925) + df['left'] = df.date - datetime.timedelta(days=0.5) + df['right'] = df.date + datetime.timedelta(days=0.5) + df = df.set_index(['date']) + df.sort_index(inplace=True) + if distribution == 'Smoothed': + window, order = 51, 3 + for key in STATISTICS: + df[key] = savgol_filter(df[key], window, order) + + return ColumnDataSource(data=df) + +def make_plot(source, title): + plot = figure(x_axis_type="datetime", plot_width=800, tools="", toolbar_location=None) + plot.title.text = title + + plot.quad(top='record_max_temp', bottom='record_min_temp', left='left', right='right', + color=Blues4[2], source=source, legend="Record") + plot.quad(top='average_max_temp', bottom='average_min_temp', left='left', right='right', + color=Blues4[1], source=source, legend="Average") + plot.quad(top='actual_max_temp', bottom='actual_min_temp', left='left', right='right', + color=Blues4[0], alpha=0.5, line_color="black", source=source, legend="Actual") + + # fixed attributes + plot.xaxis.axis_label = None + plot.yaxis.axis_label = "Temperature (F)" + plot.axis.axis_label_text_font_style = "bold" + plot.x_range = DataRange1d(range_padding=0.0) + plot.grid.grid_line_alpha = 0.3 + + return plot + +def update_plot(attrname, old, new): + city = city_select.value + plot.title.text = "Weather data for " + cities[city]['title'] + + src = get_dataset(df, cities[city]['airport'], distribution_select.value) + source.data.update(src.data) + +city = 'Austin' +distribution = 'Discrete' + +cities = { + 'Austin': { + 'airport': 'AUS', + 'title': 'Austin, TX', + }, + 'Boston': { + 'airport': 'BOS', + 'title': 'Boston, MA', + }, + 'Seattle': { + 'airport': 'SEA', + 'title': 'Seattle, WA', + } +} + +city_select = Select(value=city, title='City', options=sorted(cities.keys())) +distribution_select = Select(value=distribution, title='Distribution', options=['Discrete', 'Smoothed']) + +df = pd.read_csv(join(dirname(__file__), 'data/2015_weather.csv')) +source = get_dataset(df, cities[city]['airport'], distribution) +plot = make_plot(source, "Weather data for " + cities[city]['title']) + +city_select.on_change('value', update_plot) +distribution_select.on_change('value', update_plot) + +controls = column(city_select, distribution_select) + +curdoc().add_root(row(plot, controls)) +curdoc().title = "Weather" \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/sql_interface.py b/jwql/instrument_monitors/miri_monitors/data_trending/sql_interface.py new file mode 100755 index 000000000..613eaa1ff --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/sql_interface.py @@ -0,0 +1,186 @@ +import sqlite3 +from sqlite3 import Error +import mnemonics as m + + + +def create_connection(db_file): + """Sets up a connection or builds database + Parameters + ---------- + db_file : string + represents filename of database + Return + ------ + conn : DBobject or None + Connection object or None + """ + try: + conn = sqlite3.connect(db_file) + print('Connected to database "{}"'.format(db_file)) + return conn + except Error as e: + print(e) + return None + + + +def close_connection(conn): + '''Closes connection to database + Parameters + ---------- + conn : DBobject + Connection object to be closed + ''' + conn.close() + print('Connection closed') + + + +# data schould have following format: +# data = ( float : start_time, +# float : end_time, +# int : value_count, +# float : average, +# float : deviation ) +def add_data(conn, mnemonic, data): + + c = conn.cursor() + + #check if data already exists (start_time) + c.execute('SELECT id from {} WHERE start_time= {}'.format(mnemonic, data[0])) + temp = c.fetchall() + + if len(temp)==0: + c.execute('INSERT INTO {} (start_time,end_time,data_points,average,deviation) \ + VALUES (?,?,?,?,?)'.format(mnemonic),data) + conn.commit() + else: + print('data already exists') + + +def add_wheel_data(conn, mnemonic, data): + + c = conn.cursor() + + #check if data already exists (start_time) + c.execute('SELECT id from {} WHERE timestamp= {}'.format(mnemonic, data[0])) + temp = c.fetchall() + + if len(temp)==0: + c.execute('INSERT INTO {} (timestamp, value) \ + VALUES (?,?)'.format(mnemonic),data) + conn.commit() + else: + print('data already exists') + + + +def query_data(conn, mnemonic, column): + """Requests the database for given column + Parameters + ---------- + conn : database object + represents link to database + mnemonic : str + Contains the name of the requested mnemonic + column : list + contains names of columns which are requested + Returns + ------- + list + contains queried data + """ + c = conn.cursor() + + data = [] + + c.execute(' SELECT start_time, average, deviation FROM {} ORDER BY start_time'.format(mnemonic, column)) + + data=c.fetchall() + + return data + +def query_pos(conn, mnemonic, column): + """Requests the database for given column + Parameters + ---------- + conn : database object + represents link to database + mnemonic : str + Contains the name of the requested mnemonic + column : list + contains names of columns which are requested + Returns + ------- + list + contains queried data + """ + c = conn.cursor() + + data = [] + + c.execute(' SELECT timestamp, value FROM {} ORDER BY timestamp'.format(mnemonic, column)) + + data=c.fetchall() + + return data + +def main(): + ''' Creates SQLite database with tables.... + ''' + + while True: + name = input("choose name and location of database:") + conn = create_connection(name) + + if conn != None: + break + else: + pass + + c=conn.cursor() + + for mnemonic in m.mnemonic_set_database: + try: + c.execute('CREATE TABLE IF NOT EXISTS {} ( \ + id INTEGER, \ + start_time REAL, \ + end_time REAL, \ + data_points REAL, \ + average REAL, \ + deviation REAL, \ + performed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,\ + PRIMARY KEY (id));'.format(mnemonic)) + except Error as e: + print('e') + + for mnemonic in m.mnemonic_wheelpositions: + try: + c.execute('CREATE TABLE IF NOT EXISTS {} ( \ + id INTEGER, \ + timestamp REAL, \ + value REAL, \ + performed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,\ + PRIMARY KEY (id));'.format(mnemonic)) + except Error as e: + print('e') + + print("Database initial setup complete") + + conn.commit() + close_connection(conn) + +# +# +# +# +if __name__ == "__main__": + main() + print("sql_interface.py done") + + + + + + diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/test_query_data.py b/jwql/instrument_monitors/miri_monitors/data_trending/test_query_data.py new file mode 100755 index 000000000..7c8a41dff --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/test_query_data.py @@ -0,0 +1,79 @@ +import sql_interface as sql +from bokeh.plotting import figure, output_file, show +from bokeh.models import BoxAnnotation + +import numpy as np + +from sklearn.linear_model import LinearRegression +from sklearn.isotonic import IsotonicRegression +from sklearn.utils import check_random_state +from sklearn import metrics + +from astropy.time import Time + +############################################### + +db_file = "miri_database.db" +conn = sql.create_connection(db_file) + + +query_name = 'IGDP_MIR_LW_V_VDDUC' + +columns = ('start_time, average, deviation') +queried_data = sql.query_data(conn,query_name, columns) + +x=[] +y=[] + +for item in queried_data: + x.append(item[0]) + y.append(item[1]) +#################################################### + +X=np.array(x) +Y=np.array(y) + +ir = IsotonicRegression() +y_ = ir.fit_transform(X,Y) + +lr = LinearRegression() +lr.fit(X[:, np.newaxis], Y) # x needs to be 2d for LinearRegression +y_pred = lr.predict(X[:, np.newaxis]) +# output to static HTML file +output_file("lines.html") + +# create a new plot with a title and axis labels +p = figure( tools = "pan,box_zoom,reset,save", \ + title = query_name, \ + x_axis_label = 'DOY', y_axis_label='data') + +p.background_fill_color = "#efefef" +p.xgrid.grid_line_color = None +#p.add_layout(BoxAnnotation(top=0.23, fill_alpha=0.1, fill_color='red', line_color='red')) +#p.add_layout(BoxAnnotation(bottom=0.24, fill_alpha=0.1, fill_color='red', line_color='red')) + + +# add a line renderer with legend and line thickness +p.scatter(x, y, color='red', legend = "Data points") +p.line(x, y_, color='green', legend= "Isotonic regression") +p.line(X, y_pred, color='blue', legend= "Lin. regression") + +print('Mean Absolute Error:', metrics.mean_absolute_error(Y, y_pred)) +print('Mean Squared Error:', metrics.mean_squared_error(Y, y_pred)) +print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(Y, y_pred))) +#p.line(x, y1, legend="deviation", line_color="orange", line_dash="4 4") + +p.legend.location = "bottom_right" +p.legend.click_policy="hide" + +# show the results +show(p) + +print('end') + + + + + + + diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/test_query_pos.py b/jwql/instrument_monitors/miri_monitors/data_trending/test_query_pos.py new file mode 100755 index 000000000..42d643681 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/test_query_pos.py @@ -0,0 +1,79 @@ +import sql_interface as sql +from bokeh.plotting import figure, output_file, show +from bokeh.models import BoxAnnotation + +import numpy as np + +from sklearn.linear_model import LinearRegression +from sklearn.isotonic import IsotonicRegression +from sklearn.utils import check_random_state +from sklearn import metrics + +from astropy.time import Time + +############################################### + +db_file = "miri_database.db" +conn = sql.create_connection(db_file) + + +query_name = 'IMIR_HK_FW_POS_RATIO_F1065C' + +columns = ('timestamp, value') +queried_data = sql.query_pos(conn, query_name, columns) + +x=[] +y=[] + +for item in queried_data: + x.append(item[0]) + y.append(item[1]) +#################################################### + +X=np.array(x) +Y=np.array(y) + +ir = IsotonicRegression() +y_ = ir.fit_transform(X,Y) + +lr = LinearRegression() +lr.fit(X[:, np.newaxis], Y) # x needs to be 2d for LinearRegression +y_pred = lr.predict(X[:, np.newaxis]) +# output to static HTML file +output_file("lines.html") + +# create a new plot with a title and axis labels +p = figure( tools = "pan,box_zoom,reset,save", \ + title = query_name, \ + x_axis_label = 'DOY', y_axis_label='data') + +p.background_fill_color = "#efefef" +p.xgrid.grid_line_color = None +#p.add_layout(BoxAnnotation(top=0.23, fill_alpha=0.1, fill_color='red', line_color='red')) +#p.add_layout(BoxAnnotation(bottom=0.24, fill_alpha=0.1, fill_color='red', line_color='red')) + + +# add a line renderer with legend and line thickness +p.scatter(x, y, color='red', legend = "Data points") +p.line(x, y_, color='green', legend= "Isotonic regression") +p.line(X, y_pred, color='blue', legend= "Lin. regression") + +print('Mean Absolute Error:', metrics.mean_absolute_error(Y, y_pred)) +print('Mean Squared Error:', metrics.mean_squared_error(Y, y_pred)) +print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(Y, y_pred))) +#p.line(x, y1, legend="deviation", line_color="orange", line_dash="4 4") + +p.legend.location = "bottom_right" +p.legend.click_policy="hide" + +# show the results +show(p) + +print('end') + + + + + + + diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/trash.py b/jwql/instrument_monitors/miri_monitors/data_trending/trash.py new file mode 100755 index 000000000..bd1f5598c --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/trash.py @@ -0,0 +1,275 @@ + + + +def update_task(conn, task): + """ + update priority, begin_date, and end date of a task + :param conn: + :param task: + :return: project id + + """ + sql = ''' UPDATE tasks + SET priority = ? , + begin_date = ? , + end_date = ? + WHERE id = ?''' + + cur = conn.cursor() + cur.execute(sql, task) + +def create_project(conn, project): + """ + Create a new project into the projects table + :param conn: + :param project: + :return: project id + """ + sql = ''' INSERT INTO projects(name,begin_date,end_date) + VALUES(?,?,?) ''' + cur = conn.cursor() + cur.execute(sql, project) + return cur.lastrowid + +def create_task(conn, task): + """ + Create a new task + :param conn: + :param task: + :return: + """ + + sql = ''' INSERT INTO tasks(name,priority,status_id,project_id,begin_date,end_date) + VALUES(?,?,?,?,?,?) ''' + cur = conn.cursor() + cur.execute(sql, task) + return cur.lastrowid + +def add_data(conn, name, dataset): + + cur=conn.cursor() + cur.execute('INSERT INTO {} (start_time,end_time,datapoints,average,deviation) \ + VALUES (?,?,?,?,?)'.format(name),dataset) + conn.commit() + + + + + return cur.lastrowid + +def delete_task(conn, id): + """ + Delete a task by task id + :param conn: Connection to the SQLite database + :param id: id of the task + :return: + """ + sql = 'DELETE FROM tasks WHERE id=?' + cur = conn.cursor() + cur.execute(sql, (id,)) + + +def delete_all_tasks(conn): + """ + Delete all rows in the tasks table + :param conn: Connection to the SQLite database + :return: + """ + sql = 'DELETE FROM tasks' + cur = conn.cursor() + cur.execute(sql) + + + +def select_all_tasks(conn): + """ + Query all rows in the tasks table + :param conn: the Connection object + :return: + """ + cur = conn.cursor() + cur.execute("SELECT * FROM tasks") + + rows = cur.fetchall() + + for row in rows: + print(row) + + + +import functions as func +import statistics +import sqlite3 + +import mnemonics as mn +import sql_interface as sql +import condition as cond + +#module does: +# +# +# +# + +#create filename string +directory='/home/daniel/STScI/trainigData/set_1_15min/' + +filenames=[ +'imir_190130_otis229FOFTLM2019030204146194.CSV', +'imir_190130_otis240FOFTLM2019030210631185.CSV', +'imir_190130_otis230FOFTLM2019030204240886.CSV', +'imir_190130_otis241FOFTLM2019030210651672.CSV', +'imir_190130_otis231FOFTLM2019030204334644.CSV', +'imir_190130_otis242FOFTLM2019030210728909.CSV', +'imir_190130_otis232FOFTLM2019030204455835.CSV', +'imir_190130_otis243FOFTLM2019030210744062.CSV', +'imir_190130_otis233FOFTLM2019030204521412.CSV', +'imir_190130_otis244FOFTLM2019030210809362.CSV', +'imir_190130_otis234FOFTLM2019030204555665.CSV', +'imir_190130_otis245FOFTLM2019030210828095.CSV', +'imir_190130_otis235FOFTLM2019030204617145.CSV', +'imir_190130_otis246FOFTLM2019030210852965.CSV', +'imir_190130_otis236FOFTLM2019030204651604.CSV', +'imir_190130_otis247FOFTLM2019030210914141.CSV', +'imir_190130_otis237FOFTLM2019030204712019.CSV', +'imir_190130_otis248FOFTLM2019030210940944.CSV', +'imir_190130_otis238FOFTLM2019030204738855.CSV', +'imir_190130_otis249FOFTLM2019030211002524.CSV', +'imir_190130_otis239FOFTLM2019030204805611.CSV', +'imir_190130_otis250FOFTLM2019030211032094.CSV'] + + +def once_a_day_routine(path, conn): + + #read data in file "path" and return dictionary with mnemonic and astropy table + m = func.mnemonics(path) + + #prepare condition for data retrieval + con_set_1 = [ \ + cond.equal(m.mnemonic('IMIR_HK_IMG_CAL_LOOP'),'OFF'), \ + cond.equal(m.mnemonic('IMIR_HK_IFU_CAL_LOOP'),'OFF'), \ + cond.equal(m.mnemonic('IMIR_HK_POM_LOOP'),'OFF'), \ + cond.smaller(m.mnemonic('IMIR_HK_ICE_SEC_VOLT1'),1), \ + cond.greater(m.mnemonic('SE_ZIMIRICEA'),0.2)] + #setup condition + condition_1=cond.condition(con_set_1) + + + for mnemonic_nam in mn.mnemonic_cond_1: + + dataset=func.extract_useful_data(condition_1, m.mnemonic(mnemonic_nam)) + + if dataset is None: + print('no data for {}'.format(mnemonic_nam)) + else: + sql.add_data(conn,mnemonic_nam, dataset) + + del condition_1 + + print('Values for condition 1 extracted: {}'.format(filename)) + + # + # + # + # + # + # + # + + #prepare condition for data retrieval + con_set_2 = [ \ + cond.greater(m.mnemonic('SE_ZIMIRFPEA'),0.5), \ + cond.equal(m.mnemonic('IGDP_IT_MIR_IC_STATUS'),'DETECTOR READY'), \ + cond.equal(m.mnemonic('IGDP_IT_MIR_LW_STATUS'),'DETECTOR READY')] + #setup condition + condition_2=cond.condition(con_set_2) + + for mnemonic_nam in mn.mnemonic_cond_2: + + dataset=func.extract_useful_data(condition_2, m.mnemonic(mnemonic_nam)) + + if dataset is None: + print('no data for {}'.format(mnemonic_nam)) + else: + sql.add_data(conn,mnemonic_nam, dataset) + + del condition_1 + + print('Values for condition 2 extracted: {}'.format(filename)) + + + + +# create initial database or check if all tables are setup +db_file = "MIRI_mnemonics.db" +conn = sql.create_connection(db_file) +sql.create_database(conn, mn.mnemonic_set) + +path = directory + filenames[0] +once_a_day_routine(path,conn) + +sql.close_connection(conn) +print('programm end') +##################################### +#programm end +def extract_useful_data(condition, mnemonic): + + temp = [] + + #look for all values that fit to the given conditions + for element in mnemonic: + # print('at %6.8f appears %6.8f',element['Primary Time'],element['EU Value'])) + if condition.state( float(element['Secondary Time'])): + + print("condition true: value= {}, time= {}".format(str(element['EU Value']),str(element['Secondary Time']))) + + temp.append(float(element['EU Value'])) + + length= len(temp) + + + if length > 2: + average = sum(temp) / length + deviation=statistics.stdev(temp) + del temp[:] + return (float(mnemonic['Secondary Time'][0]), float(mnemonic['Secondary Time'][-1]), length, average, deviation) + else: + return None + + + + +def extract_filterpos(condition, ratio_mnem, pos_mnem): + '''Extracts to filterpositions corresponding ratio values + Parameters + ---------- + condition : object + conditon object that holds one or more subconditions + ratio_mem : AstropyTable + holds ratio values of one specific mnemonic + pos_mem : AstropyTable + holds pos values of one specific mnemonic + Return + ------ + pos_values : dict + holds ratio values and times with corresponding pos label + ''' + + pos_values = defaultdict(list) + + #do for every position in mnemonic + for pos in pos_mnem: + + interval = condition.get_interval(pos['time']) + + if interval is not None: + cur_pos_time = pos['time'] + filtername = pos['value'] + + for ratio in ratio_mnem: + + if (ratio['time'] == cur_pos_time) and (abs(float(ratio['value'])) > 0.1): + if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): + pos_values[pos['value']].append( ( ratio['time'], ratio['value']) ) + break + return pos_values diff --git a/jwql/utils/edb.py b/jwql/utils/edb.py new file mode 100644 index 000000000..749f9c156 --- /dev/null +++ b/jwql/utils/edb.py @@ -0,0 +1,40 @@ +#! /usr/bin/env python +"""Tests for the ``engineering_database`` module. + +Authors +------- + + - Johannes Sahlmann + + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to ``stdout``): + + :: + + pytest -s test_edb_interface.py +""" + +from astropy.time import Time +import jwql.utils.engineering_database as edb + +def test_query_single_mnemonic(): + """Test the query of a mnemonic over a given time range.""" + + mnemonic_identifier = 'SA_ZFGOUTFOV' + start_time = Time(2016.0, format='decimalyear') + end_time = Time(2020.1, format='decimalyear') + + mnemonic = edb.query_single_mnemonic(mnemonic_identifier, start_time, end_time) + print(mnemonic) + +def main(): + data, meta = edb.get_all_mnemonic_identifiers() + print(data) + test_query_single_mnemonic() + +if __name__ == "__main__": + main() From ef28f4e7bc390b96bac950f457df1898c38f41fd Mon Sep 17 00:00:00 2001 From: DanielKuebi Date: Fri, 15 Feb 2019 14:54:41 -0500 Subject: [PATCH 010/174] added some plotting functions --- .gitignore | 3 +- .../data_trending/15min_to_db.py | 46 ++- .../data_trending/MIRI_mnemonics.db | Bin 401408 -> 0 bytes .../{ => data_trending}/__init__.py | 0 .../miri_monitors/data_trending/condition.py | 205 +++++++------ .../data_trending/csv_to_AstropyTable.py | 60 ++-- .../data_trending/data_extract.py | 119 ++++---- .../miri_monitors/data_trending/day_to_db.py | 113 +++---- .../data_trending/filterpos_extract.py | 112 ------- .../miri_monitors/data_trending/mnemonics.py | 32 +- .../data_trending/plot_functions.py | 18 ++ .../data_trending/plots/__init__.py | 0 .../data_trending/plots/plot_functions.py | 18 ++ .../data_trending/plots/power_ice_plot.py | 72 +++++ .../data_trending/plots/volt4_plot.py | 63 ++++ .../miri_monitors/data_trending/plotting.py | 92 ------ .../data_trending/test_query_data.py | 27 +- .../data_trending/test_query_pos.py | 79 ----- .../miri_monitors/data_trending/trash.py | 275 ------------------ .../miri_monitors/data_trending/volt1-3.html | 60 ++++ .../miri_monitors/data_trending/volt1-3.py | 64 ++++ jwql/tests/test_edb_interface.py | 46 --- jwql/website/db.sqlite3 | Bin 139264 -> 139264 bytes 23 files changed, 597 insertions(+), 907 deletions(-) delete mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/MIRI_mnemonics.db rename jwql/instrument_monitors/miri_monitors/{ => data_trending}/__init__.py (100%) delete mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/filterpos_extract.py create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/plot_functions.py create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/plots/__init__.py create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/plots/plot_functions.py create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/plots/power_ice_plot.py create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/plots/volt4_plot.py delete mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/plotting.py delete mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/test_query_pos.py delete mode 100755 jwql/instrument_monitors/miri_monitors/data_trending/trash.py create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/volt1-3.html create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/volt1-3.py delete mode 100644 jwql/tests/test_edb_interface.py diff --git a/.gitignore b/.gitignore index cfad500f7..88e834c15 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ docs/build/ docs/source/api/ .cache *.pytest_cache -htmlcov/ \ No newline at end of file +htmlcov/ +jwql/instrument_monitors/miri_monitors/data_trending/miri_database.db diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/15min_to_db.py b/jwql/instrument_monitors/miri_monitors/data_trending/15min_to_db.py index 9b40cc98b..aa0d48a0b 100755 --- a/jwql/instrument_monitors/miri_monitors/data_trending/15min_to_db.py +++ b/jwql/instrument_monitors/miri_monitors/data_trending/15min_to_db.py @@ -9,9 +9,9 @@ #create filename string -directory = '/home/daniel/STScI/trainigData/set_1_15min_complete/' +directory = '/home/daniel/STScI/trainigData/set_1_15min/' -filenames1 = [ +filenames = [ 'imir_190130_otis229FOFTLM2019030204146194.CSV', 'imir_190130_otis230FOFTLM2019030204240886.CSV', 'imir_190130_otis241FOFTLM2019030210651672.CSV', @@ -19,44 +19,32 @@ 'imir_190130_otis231FOFTLM2019030204334644.CSV', 'imir_190130_otis233FOFTLM2019030204521412.CSV', 'imir_190130_otis242FOFTLM2019030210728909.CSV', -'imir_190130_otis232FOFTLM2019030204455835.CSV', +'imir_190130_otis232FOFTLM2019030204455835.CSV', 'imir_190130_otis243FOFTLM2019030210744062.CSV', -'imir_190130_otis233FOFTLM2019030204521412.CSV', +'imir_190130_otis233FOFTLM2019030204521412.CSV', 'imir_190130_otis244FOFTLM2019030210809362.CSV', -'imir_190130_otis234FOFTLM2019030204555665.CSV', +'imir_190130_otis234FOFTLM2019030204555665.CSV', 'imir_190130_otis245FOFTLM2019030210828095.CSV', -'imir_190130_otis235FOFTLM2019030204617145.CSV', +'imir_190130_otis235FOFTLM2019030204617145.CSV', 'imir_190130_otis246FOFTLM2019030210852965.CSV', -'imir_190130_otis236FOFTLM2019030204651604.CSV', +'imir_190130_otis236FOFTLM2019030204651604.CSV', 'imir_190130_otis247FOFTLM2019030210914141.CSV', -'imir_190130_otis237FOFTLM2019030204712019.CSV', +'imir_190130_otis237FOFTLM2019030204712019.CSV', 'imir_190130_otis248FOFTLM2019030210940944.CSV', -'imir_190130_otis238FOFTLM2019030204738855.CSV', +'imir_190130_otis238FOFTLM2019030204738855.CSV', 'imir_190130_otis249FOFTLM2019030211002524.CSV', 'imir_190130_otis239FOFTLM2019030204805611.CSV'] -filenames = [ -'imir_190130_otis229FOFTLM2019030204146194.CSV', -'imir_190130_otis234FOFTLM2019030204555665.CSV', -'imir_190130_otis230FOFTLM2019030204240886.CSV', -'imir_190130_otis235FOFTLM2019030204617145.CSV', -'imir_190130_otis231FOFTLM2019030204334644.CSV', -'imir_190130_otis236FOFTLM2019030204651604.CSV', -'imir_190130_otis232FOFTLM2019030204455835.CSV', -'imir_190130_otis237FOFTLM2019030204712019.CSV', -'imir_190130_otis233FOFTLM2019030204521412.CSV', -'imir_190130_otis238FOFTLM2019030204738855.CSV'] - -def process_file(conn, path): +def process_file(conn, path): m_raw_data = apt.mnemonics(path) cond1, cond2 = once_a_day_routine(m_raw_data) - for key, value in cond1.items(): + for key, value in cond1.items(): m = m_raw_data.mnemonic(key) - if key == "SE_ZIMIRICEA": + if key == "SE_ZIMIRICEA": length = len(value) mean = statistics.mean(value) deviation = statistics.stdev(value) @@ -64,7 +52,7 @@ def process_file(conn, path): dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) sql.add_data(conn, "SE_ZIMIRICEA_IDLE", dataset) - elif key == "IMIR_HK_ICE_SEC_VOLT4": + elif key == "IMIR_HK_ICE_SEC_VOLT4": length = len(value) mean = statistics.mean(value) deviation = statistics.stdev(value) @@ -78,7 +66,7 @@ def process_file(conn, path): deviation = statistics.stdev(value) dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) - sql.add_data(conn, key, dataset) + sql.add_data(conn, key, dataset) for key, value in cond2.items(): length = len(value) @@ -86,7 +74,7 @@ def process_file(conn, path): deviation = statistics.stdev(value) dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) - sql.add_data(conn, key, dataset) + sql.add_data(conn, key, dataset) def main(): @@ -100,5 +88,5 @@ def main(): sql.close_connection(conn) print("done") -if __name__ == "__main__": - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/MIRI_mnemonics.db b/jwql/instrument_monitors/miri_monitors/data_trending/MIRI_mnemonics.db deleted file mode 100755 index 162be971209e71b5b2d868981b7b76ea10470855..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 401408 zcmeEv2Urxz);2+cWK0A>QHcfwhV0-Oa)u!)B9cW6fQXm_k`Z&xV$Pzj*;UM#5ffro zz=Vo9>t79{FkMh{?_KWq?A^cSd6vbVemPZdpYEBr&Z%<({Ji2*Vs!>4CJ&2D(cy@x zh>44f@pN>=#3a9oiHSv_|9-vv(Jw82{R919yut5H{EGgDvCbkX^sbWf0p^?t<*&*g zlwT`XD?e6#sC-AcO!k38K4YM1}FoR0m=Yn zfHLs^F9RXoXzl^-=!drx`r!iT2iTw=)*ST1tOxqh%@F<2?TmhCYoQ-%?a&WZRrEtn z0sWAbRzyGB$g7DdA61^B?5NyA>7-J&lABUX#j}c26}=P{6)q~wP~a=5%3qP6D<6bT zq%X<k38K4YM1}FoR0m{HXj{$BQSuq(IaTys7ubR)7>U6V;`G1gpk`Es)YHg7< z)vtv4$78lS<_UfZyOH&2r&j5%1<#82#WKZH zFgVBDPN5#oFBDiWtc4#&#K*3+JDvd^(Qd`A(a9Lx%-l}C9`09D2luO~h2MY+7rrm= zfA2GRE`8WZ>)~}4+|t&<#!jvt&Yvl;KC>47CtTRHFLz){7-D|)r=;LD7~I^(u4O&k zuS8(Iq!zv&Szmak*TyvPoZ?Vou}l?%b8PHn>*4$)f%T+X_yc6UYuckJJHYKy*^j~Z z3NW~tjh#$A+^3i#1l;GgK9#M#K^SgkV{2-`m1f|-?l*yTJ_8Se zOLOzQZd-@L2J6dDYgRjBXcJ32sRqUc%=yi%HC}z%*QZELt1*cs+4T3SanU$Sn12|tW_1dwEZG#KDBMRQVItU)c zx8?Q?jlIo-+dH`3DJaq5)KxSoM5IJ#p@?ZoThz9n^V-;!E* z8G;WVDCHXgo}I?~&4}-g!Oc00{zpN7`l1X_1}FoR0m=YnfHFWCpbSt3Cd`vk**;!dysaPpb$xBH|u|#o}Vt`^>g&PWs6halW5DQz8K4YM z1}FoR0m=YnfHLq;XF#|bV9O!8Lw;2Q4B@rv%=L*pZwpbu&TGu7gBq6D5>x}^)>Z?o zt8FG+4RA0*FY9gM?FNy>Tl*e2c-RmvCT+^Q&OJrw zfAxCuqoXGug@D6m~y-V9Y>RAELQhvx06v{?NXL|G>; zxox)^LvzfzwRHdmwhIOGE@aR{(a5{`Z`!mO`X}n$yqnw&Lz|g%Ys&!Ep@qu;MkCgY z$9I%D29Y|WB{t_eU}#fwLKVPFWPH0yhx4)!G1t7aSj)RCxTUSR4Y#%mpuo6*Hosch zQjqcC3Wd{pLwNj|l5MBTF*L`9TU!KBF!3b8#FLnbM<8^VHZ$>v&1#QaU#DPbGaGJg z4M2fy0d0P@v?U?*g__y!deA?W*Z1P0F&Ns^rs)!Z{g|7q!pX@AqPQ&wCVj{eMxz3P zCYIdV5`Y5RGwViPR|0T2vK_6yS~~?I`Ys)_U5SkrQ2{Uq8IMhikB^7Qc?U9lDq6FQ zqk^55gbIL@(KY*U4~+*RM;gFMZFLNeZj{;zfP%4`2*z&0us#-zUGBKvNpzz`R(-Qj zU(Z4_1pv9V1pw>N!UX{J(2a5;g0bDV=+)`Nm8@c+W>%=yD5L+S#azS`ljWYt+Dk2z zc!mDan;FBu_>yJL%hZ>aQM0lV&mEEy88tN4-P_$i+|4uG-O)Kbz}Yc8*w-u2ywQU) zj{eT}fzCRC_6}aoI*p#9W2p1*zT#ta+nAp_#$dvfRgugTz9i61u zWTpXSY)m-nK^f@o?Hmwj@6Fe7a(1x~@(R>(4D$DP_6ZFC{m#F9A^868-uC_>I-brU zI)?EvMn;+XQlcD{)0h`>q(wR^hdS!-n}<4ToeTdJM{Oa>QOz22R5K}&j%r37l{#uO zK2aq_IjU)6j%wOMq@$WrN2QM1e2yw5A*0r*lXz}W?RV4w=kR_GK>@*Df%U&h3b|zc zaq5TE51Y>q<;7*x^z_7YLw@@q^L#{F$)54JU%yLWzWD1M>W$PJo6Q^R;{S$q@qZU( zS*5c|`AR-YDvG6wa}|RX)fLJWmMRQT(2;*AzeawbJkw2(z9<8f0m=YnfHFWCpbSt3 zCJR79%uo6R)QUDNZ1H^ku{(J$&cGFGQwI0txlT!e+c|A2W zPWNA`1Eqei`X=~AVQ6#nrc(eq@QimHp0V!URJd|!W0d6$dkjrV0YD=^HgxO74*^iD zcif@Zi^e(uHk|^X#p}rrd6#%U5iT#NvR@i;AG2+0&aG_(SQpV1ZUm^wGcGyxCVkss zxboU%(x{XNEHpU-K!ewF>}aty*K**LM4Q|1wRd7@QU-tyuV;LhB^M{1hntDpYaa35>7r$Ugc4G7DJm` za%)=v3ebG!VaTiY<1-O_bzV#EZgMxzYed5x|NyKxuH;E@2eQsg_We!ApUP9CbvOWQz~ELkzyY)E3uxW$9Y}A z-$+45&BjJNcUWD(-_?m9&IJ9Lh<_l*t53M2e~7Ppqeqp67PcBchc0o^C9Y;$;>y&Q z6N*p?d8H}85myx)R3vz+;1uep)KQzyQRQ0-MFs-H`vp3hF|h&rfCj%S2)Sj$cc?c~ zZ)`?yl*PT#w1GF){Vv~#H`cvFy^(rj^Lb-;85y-+y~J}9@h_ZAuBU%^ptCn$=-Xud z;g&)^+QdoJU8%b^pS$YIH~pQ{%)gQEoPvW&O}lEK;1uep)KQzwQEB}@(pm?7k1{|R zpbSt3C%@pgsl{>0#< z{r?U;b=hT=)6*Pa#N@NV`=mBvXwv?Fgl_MtV-VK^5>CzWO7LC4LX-FZqltgWt8gkZ zhGc`X;q(98fT2nI|2=r>>fSv{I7=b9OR&=h@1q!+wEsVtr}nWh_tgAQNa$j^YQu88 zfdHumKrl~T=USJ;dEt=!XZ6rq?QAgHr2YRM$k}8*-<&iDhP(875It!e3r*ht-xtkW z^NZXoCm5d7{zK`_A`DI1|Ib77F8^_H)p8hqaf+R9NpMXYeWU4E6U94Ci5P%P164nNdFJk zlGW)2sXdkszN)()LzDFXSY&&?o%X64828hzN7$ja!e~bSTQ%JQ5Zxxubmg`$y94(xvOK8$*Ka9B>?k*tyb>i^e9Uaz74 zpQ_}oq^x*ZagJh;qME`@g(V7M3LWL|%de7;K_}7|Wq>k38K4YM1}FoR0m=Yn;GfQb zU7PwCI-lRL_`g^_Pu=K#&f^YBklD(AZnce20AHm1kL9WN4hk~vzY5Z;j&8PH?j!sS zU4;A}%2V&WI^HMz31s&5jQrkz5QZko|7$$8vSn%OCEr1Qo8ihZ`#A89*O6LR>U zxnftmgS+g8jG7ef^-CXMBR93NlWusI)s38Am#H!z-DPP5@^4l^#`iU517%JM!$s(S zL|dnwvpg~^A$@Skj=|MaS!ksHt(va?&)j9@L)|L&!_-8(yMMIYj-g5VAGu2p?H%RD zZy?)$Oz3FM^%$BI{~yd#*I%EuIxiKnrQEOFdXUCKll4EcZ7kEq%u5Y2dj6?f63%)) z{I35`_5VNp1B}j(GC&!i3{VCr1C#;E0A=9cVt}Ckry?1AGFNM26{OEt@ysq+rQtX9 zI{k0WZCx+P`~|(m{F4|-hD4Q9(A~R--68Y(*=f_lp&?pi`aerOFTa2cg#(tC zJ`BUqME!q}r*?bA*q_CZAHj#Mx?V`Zpj;K0V(>C~W`HDyCh31Ly3F|@e}SGL|&`2ttv*ShK;p1Q91 z3hBAMAS1!s*vv=kH?3q4;8|c+!f!tv|}y!{MP?c5(Q#1Kc)B*1&V%hnhxD~-&f1gC%mAY&5xvP<4(|vr+0~+h&`|GF*!C8NuLw%L{YV-MO7gZTG z3k&hwc)SImOCz2tIM%jlKNXxoy_9-s^Lc4^m8QP{Sh@Q;Hu44F_u)28`)c6tlc>8= zcWpLzt*ie>^Z)-#5|;j1$^d16GC&!i3{VCr1C)XPMFt4*|9qZ$i(HTV*aFBN72vu^ zUubW!$oT(2p1Q#ayXro7A!GQ%xpRl%JMc-{|4}%&#biz0eSDbWzA!#{qOx#w5bYiy z#{bXq)Q**Hns}=VWb)g|T-#QK!AbFdH1yU-hQA36hba#aXK3~OfT2n8f0UAKbLUTo zPbM&>s{ev(QW7jQ3T#?%mFfeVd_iE7UmMs&Xz}yk?;UywX|7{x9+oF#Xj1$ig+(p6 zPLD3?!j#mB?Iw0I$IztsKME#G&Hqra+8Z)1esK86eTAV(@qc8Sr#EY|);!3tcYOOb z_YHveXg`r8?{}F2vnmN5MjDs}KFVmu4 zea6tl`2SpFJbw3pQ67-3sFgHyca|_*MEoDc;H97G%bxOrG}8=^s)$$?njHU+;;G9S zEx(}B6Q)Ey8!f#{2SbzM|0r!jHUCAxDRhTF%#?n#ej$b?#s3jnexzHL?gB_3yJFUz zfnqE)IsVVMs^R3xd(|Or=HcYiTz1w3-T(i;_+doHOc|gIPzERilmW^BWq>m9UuS@z z|MPh2Ee$V!Nm{5Ihk@_EfLk|o|HX1bsrZ^9s7jcR$s>tR4qIv4ps)?&I zO(3hC%AZDW*qf?F=>PLPwUVlW*;H!-WUf~qQXPw{Xp;U%s|V2w zi}ZiKg&fVV^A{<7#B7uFKT^@whyAX#&4Kig^@qIl@ZAF>{f`9QP_f4u8s8w(NUZWl z4|dH3D_fHON37XoKkA#I3R!RBwQuY#VU3)u{}DPmV9>{n`1j zuyGOk9}V5QAfkO_4@fJ&+{gCmCl;El{~6L1PG+t&feeWiE#rQ;VrY{7kL0P_n{>UQ zWDl94sv9)F+``Z#{g3W&?$;XK?ROx3Vvp~=|70!MF!lck`TuC-KADlGWiKFY{t8L+ zb1hi5ss8`3e{xU*lmW^BWq>k38K4YM1}FpniwqF-|1_SuhugKmJ)|Jx#Ou3%Mp`xe zjxAFEGuy#GFFReK1Zk0ht+yQS*bpsp|38w!Toe5RirJ8Ed;V$otLvP4qIW zXIoW4*4iu2Uaa=T;3WNzc9VOmo|BC@3K`uVC1e+L#?U1Fk5&(oLT{ZI-4`;dqL(}< zv1Xyk`X31!ORp%~vv(moKiT)@)*=i|(*FoO_|3xdJ2jBGYS;st5l1mJN&lmf_X_l# zx%DDsmu~)KTFOq@u(BoTe{{h|eE2xcX#`}c*!E1VnU2{e>3_zy;_R6>ro)s+o7YWC zXWJ(0|0JGzd>^}l)I6AS;=}ISC#qSt$?<>WY<7=3Pw5(fY#)rUne_-mi_rg8p?3Q| z&xiCvHy0gimJ~&q3!>ux9=euKBxE3CWxT|uk38K4YM z1}FoR0m{ICi2;KC&*!Pjw4b@I{aeUwp{4b4=lF)-vFr4|P1E)Nkcc)iR4J7(fxIU} zG9`Ahs)LKD|Bt@kOX*aO9%m0z&%6HIXvq!-i-`YU<*6Ol=~a|{9wwhST&`Fe&~WIr ziw`9IkBn<5uKm(Q5%RBd`cIB#rA3Q~|05l2dt^$V(03>nJ@LSqW5-#xMe2XCJvxC` zRUms+M~}d_h8UWp{}WL_*X~i^H9N?faB`pWo^BYLr2mnillYW$;Q9v0UFBfAesCs+ zCh32K=6x#dFxdt&dk-F_HBT2qlk`7x!BeN5T{{CZ0=67iKRJ|zChLD>+cM?-Im>9s zoHTX*tU_@NO^W}c%!L-+#o}v*L)NJ!J=?5S!q6oBk9rSCE<2~v=&{zuLxqq*_gxp36*p<_TthfypvS^uMj4@Kvr$+7z& zEo`1r{8m`8{4zk8T{n>f5=tk=eK%mqXt=nY@h7?z-l%j zBL0uuMW&m((OlGBk38K4YM1}FoRf&Xm=2>O2- zPouZ(r}3j#!qJxMVV`(E8-B^pcYX`4l zXp;U%3Lf4$K9*VtnXmpZx>>Q0g(mBNWP9+t=s@QJv{W^VGpUgdekA>mWU%Y!(PgGp zkg>J0e`p1}V5Sc&CX-q%onB&DR@%-e>8NLYZ7SpJ*2-`Roo`*6oxjp;!4(+Z1CeV z4?9%BpU;$RK(6Zh@nh!6G{|h9x^VunV;Gv0|BqG=JkMrdQl1W(ZC1Cs(no`ZCddEL z$cOlMmy9|J=~{F4^x_CTn>e-&>i@47liQ3s00bxoO01Vy@xT4BrlbF7F;Le9z^Gl5 zb*lNk-u`$|)!#?kH(gWp_c^qrDlMtnY$a7QjoLKntFC>_IJmF=KC02Qul_!V`YQF+ zX7tt81Yb2ZZ_HN(N7YHbDmaJwD)rUo^Hu#;GHO;<;<-cc`u~oOjnq~Db+BF2p8D$) z>ZsIFo6k{=+BW%6MPE7j_(S#gQO%}Z_4hf{SE;Wyo3GOP|NkttOlLzGpbSt3CiLSFtnKkSH6DUel>OT_N!s$jcofHddY8zfy^~$HSdiy!O*1mKbm)&r>ddpS0Hb= z;lj{@C>EL=|3{_hoZJIPO|^ualh4wgZD!Y85V`-qjm3~5(HkNATgyET)y0@?jtN($ z!MqCv^DboOjY83FJa(yZ%OLMaYU*j*aTr=u{=e3p!~Au}A#2CdouOYth2bLd|B-R^ zVXgb`*@|5CzIL_SLKd1F|3{>&?eBGH|9i;n88t=bPj>DBs?kY^|D!aS&ixMLN2@}% zgubQOiUQ0wDgKY15A8$dJ-q$~vfhsR*r$kJ0ljz0fpu6l;!q4#& zAh-2|O-cio2*X9_f95{v-Fdv$K39T)djnuR9oe?+>F24^sD|H(_X! z{zn&FE235QSZ~PjzP?kozY~Tg>3?*|+C@y8w^I-D9?Z9UH_n-bCg=a7^})6;l8+VX zL-yo6t7-l5-2+trH+ag^Unv8W0m=YnfHFWCpbSt3Ci@w!4e79>M!8;)J%3rBJ*hn#qDAiiN5SOJHi1*i=fRYTKeyUVbjQ#{ z{ePLKzUS`rhqlvUPRDEGo?d1ZHWv~9kLPJf4T?-oUx!+g_f^~AZJxdf49$gwN;nsgR{MlW+7q0vkC=|D&~s zilv4DPKzLW61dGPs$!wZ`hO5lLwuxr@>*rcED!Pium;Z_AnAW3WWSAc%Qa4h?2gN` zZeCiB*(T|KBxG;MI9)xr4Kl+17!8 zfHFWCpbSt3CnIIAiAjqd%F}4kVbfs;C75Cq7oYK}pYS(y5%GVNnf#^T9?vTk(lw-K=MUMBp^5tc z8c%(?!NF)Y{& zvd0+AcIA|^&}99O)Z~>ZyJ~WDA?wmxv9WX8VrY{7M}kiNQ{cGdu`s#Kuu8EH-WZys z|B;ydxb?!%{Wl?}&ry}d8!9j~N&h3;29F-=j4Ffd(04}HirKp+tZYg8A7xEE>{PL# z$2CY_-}{iX;vSZ5vi?WTCh@e~ea&ac&K>#v+*)?#g2?#)cf~I+jxK?WSH1ejgr3H1 zlk`6t`Hh-W@4Dqe?x6`13w@L@G*SOAMt7N6JA(mIkUwEun9TkZVYq1h|E|rtYc5Yw zpNz`lVY^?j&}99O(nWq;tJx=G4l}jCoc{9ZHijnY|6x20rKFDmW|fc~-&N9W*(VH5 z(*G!J;`9VeBAChLD>+i60<{DSF_vu$(wOcz#*PMZIZJ?QBH$^d16 zGC&!i3{VCr1C#;E0Ahfk|8sci9sT0JS4%+l=;QhaJ_`xDe?kANTTVDHTk{-pRLi>U zsA7jj(cWSXw^jY>fu9OH zxPP=hO#l9+;?*}j3{KMjC?2e*`p{O)8#15zg@+ta(oFnr{>W=~u4AiM_vtp-K85UGTO)S14_K1ygRD^7C_8spu$! z-@=63vVpsl2;8NFaTmn7qwd@2L2{6J{#ses95RK0t_OtWr~c~8zdz?_}$ zVQEX!|0s#Rt@q}W$Id|7rO7XbIbFtV6ZQWro_dGP+wZK%g=`zixM|~C3d2R{e`W{& zF_%{U10c)n@?kEp?h_I9|B<_Bj`DF4({Dl>uk%C_PGifegk>joY5E>#sB3th)Iie z6GPSigC&Z^x+zA>osv7uxJ|=XT@7|kO&PVGJ;igQ>*}w1yZeXpor1!H!=2n+!!3f{ z*#~rli&$6X@*f?BD%gg*1BJh?L0Wl<<`JVX->?&h}pa@gIv# zh-rquqN5WNnGzYElo+3oGU6`{C^9uRIWq1qP6{y=6Pp?znG&Cv@RvrTqmvYyJTNhN zSZqvqWQtCpySH;dpuIO=$I02nKFBLj$1$kc>R4vVYsjeS>51ot)H-K?b9g_-A6@v) z_6>jUDCCk2-=S4if8Qz88|(b^Uw$m`{oTFo{X=v-okMgC<714BGWFG)ELH99?TQxw z{&lc}kbkJrq5qY4gc5gb);3UyTKsLkf6 z=-X(AhTlkqf6vs^YT~P@y^p={SJS!!orT@E{;O%-+thWb>o%L~W@@%?)H&VRy6M+} zPK`U~ueYglQs-zbFHg0m=YnfHFWCpbSt3C#YA+KlQXKjl{HTWznxJvbrY`!3p&Hok2ZtCrJTb$UcZxyN9ZZF2p8RJqW? za>@{26v;lSHvDtp3Jgt(|D%v>%jW|Y%+rPJ3$l?jCGs#dDgKXG;|13|Ncajlj}0o^ zE-%8+r1(D?xvbX{g-L55(@te_yC_x~yomTe+Dy*5zwgrI{*Vb4K3!wj$tL9ZKN`9G zuF7Q>-EDLHE9qd%< z;li5^#8$ye&Rmb6J%QkD+I51hb1(+yaD*EF@fnIx#REQ*gzgD1dyF!jak3@2%!$dJ z^Pb;zU)v6j-0hSRaFo z&niiZIssmxhqV)C;+y|1Ozdi#{|Stj)S>-KYR5heUYZ#{b64hpZ|afa^V^JA#+j7| zq3Zuh0^>=waKxP6!mb|`Mu7YByb6=S3otmx#7?T7IX?k&es#<(MAl8x`o53@pVUF? ztH18T;ASS=7WGs26HMK&cIu-MI$^;y-wWWi<@22p)1M2YYc~(rNj5Mp7`tEH*yq4Y zlWF;B5);7FP{H2AwLb<&s|OMdjPnJ|@tLWo!b{7Sqet}Sf^VzI<}LZGr7C0``MY=n zw21(1!k`DhOLLFbOHTWP_a3iZ9Y$?r*=AM`nE1b{{DV6E@1iWLbXF-}$wx^=u~c!c zVz8pRLb<|Hg)oJV=oI>*3{VCr1C#;E0A+wOKpCJ6PzL_)3~*}~0>o>(stGRy`0`pm zy>(q_z-+KCQWz>#(`fU7+I4_BwD3BBH$tC%R%|#EtoA;<@8P>mSX|Tlfdlf>`QGw*|CORn|x5a4p297;dKB$LSG&}s@F=e++jB8L@_&Oqc-`Vb{RlG zx(eeWX#GQl(PRz?{b2HuF|WaL^xjMECuA|(91CvkDuBRtVI7)Z$eRUbg_i_*&Tsx!yOx zGCy(s{@JZDG)e!vBlO{LTa7rdZnNN+#j9KlP1662uN``HKfa>Q*ABX}&{Y33K1N@Z z0m=YnfHFWCpbSt3C;FjJsuHcqXA3>y?VCrH?F;et0jT(3llp%sZ&j&w ziViu7#w!cpeNO79JuALpaFYIS$6Iwp`cv4oYF@hw z;|}+H6_24w`X5dFQuS;p(>3r;A?;9Hw{jMmtp7t1Z9`{Crzt?q+aGanqr)*YDn8hx z{*Ontci61@&<|dZS^2(?swajf>Hl!vs>|BOgQaG}JBKl1hdAs8fMos8TyXA4pMI+F z#%Ok%JbirSf~5ZkA=}rlZuPhUZ(S6)F(#~9bak5#n$-UhXx=-I_wE=6Z|!PMl`g?! z_$2)w%Ug9s_2W(3JMi{)?z7=n#aOJ>Za!#I|3{&Dzd4gHlbimYtjYY8$*6TT-ytS*SND#}?K9r}6)P(374{g)%@HpbSt3 zC3h=MaZF2nIkD2!c%N;3T zy(99$nx7sRnxy|-knQ(lE-!fqRwlBOFABnF z5%GU#H1Drt53SUOUa~chvf`33G)e!X`2YJKG7pq)g3ZzKn>)zfz|bW9&)nd*Y-bi% zg7vEDJ^D>#mC2y`pME}21}FoR0m=YnfHFWCpbSt3{%H&}t^X5vgDf^Koy&=YvVoD^ zi>^);78fG){~+EV%LRtxPjcbLt_9mrxH)xPz+#fBs^E59`)2s~VVrY{7Pvi|UuQ2^^s1nLX9`YF7husJd#YHW-P3r$4 zyg^o*_Gsj!LfIatSxxhN zplry!GoSZ&z|f@p|G|j!7Gb3U;c(+sRqh09cGiTIElK~Qky|EgTJ{dH=5~qGX|F2F zHaY)49?kpxu^IjK;il|})7OqPT70lc{g1AgWr@TFLr=KTVoTzhiRzebQvUxiH1Byk zI_^w{vipxK4W6@Ci^%!^2yNc?TdO@s;ktWo-F^p`2-_B+|IxfHm>mrMaAV8F9e=E- z#?U1Fk8E3|cO5q%1j^djCv9-Mh@naPe>iWDjn#!hy?D40JK)s&^u;XnulT>Tn7f$V zDOr7~EQzOz(dZxc|K3Bet~;Q1XBjn9Q}Ntbc6UIgrHYSL!}frJBSww)xd;vyyiXfl z&_)-{*61QrQ>W1`EOp+7xu;dI0m(b9f~j*-=WIUb)b7~m!_&R)QP`lj>hB}^jl1XX z_o;tU|7=G8?9ixx3O=eE_~-8r_T&SrGZP6X#{@XfQ%`>cWuNzO^XdH!}y+Bf}INA>@_Q#Gys_wTKB(6cE6lmW^B zWq>k38K4YM1}Fo6YX+K*|N9iRQJ=-F@F<2}J3adDdazD7JjlfV&70o;&!C6B6<^pH zY*X*u+R`2m;1lEjxd=Yq#L{IVSg)@#EG&4088*)XLuY}Uy6|Go%4YoM}?7J^u7yJ5)XJsI=)iCmWV+a{fQV+PR&p9fH7a zpvkV_r>y>e$Tlhd&$w!O$Kd$=!0o71Jm*78%r+_ipFz)m$nPBw)-v8o96#3X0TJ0BewhcPK%2YvUQvBZ&q2arWYCq^z ztfziAyshxOY5xD;`oT>HM;V|DPzERilmW^BWq>k38TbzvXj=bA@&@@-O|MZlh0UkMHIM1zgTYQPN-$R7{M|%(a2fr-+v>q;(OZ&J)O0aDI%KxYR|NevLDuqx6CR^ruI&ECdhfe?DX#@TSLA24_yh;3WO8h|qS)RT5TUXYKe$+r|Ih{{KD*o%V24 zR}HYgd#sz*AzVe1_W%1MbieKgw*P_v{S-vJh|1&q)xW|0=Rp6M~ zHgb936=B;5P165?2t9CNPPhADFZnIi)>a8alj8r*2;Jf7`46qYUUNx*y{@lWXsZAJ zjt^`)CdvS1fHFWCpbSt3CplSU-m^YSx;NkJSW$@UmLl^V@nZoiwg#I7O z8|!!fY{I(}@MJ|B?ZI4jNY>hxsQ=4(V?(^*m}zTxl2>wcm!k}3oTUHTBI6k+Zp>1K zDsQc_LNRtdxZ1pEMOiT!nYy0r{w55AylQ_FrWRWgn)um+S9i(7<2DNxexJ^&$;Qwkpy8xh#f_a5Th_Tt$=W z|D%!nJDm>Rc?&ADigtA6u40XxtpA7e#`^w9oyNZaRcjt>YVni3RfU}YkFJ+5wFfp_pxw{zta`dzJO|+6GlubgR}_KM+QX(ErSBGHgiR8)>LIZaqA4=voX- z(*MH{de&i;OF!Uo^nuvhb=t{7Q~m!Jo~sl}8K4YM z1}FoR0m=YnfHFWC_&YMtwEpK8wLh6(5Is8;9M9^NEttZVgXI1HjtD&>Z|WE&aOjtu zygPsu7DcNEME$=6!K2ptb|?aeolkao-eK>_wzehde?^2IJhoVy3l5*fW$lXD{ou*^ z-;bHNMar@b;MB2T)ZAO_)CIEs_dw|JHR&&&fPGQS=mk&NN|qe|_eAKJL3w@Uz}Y^D z(>~#vuqH!m52XG74Cx~;j^~Sm{p@E(wj0^=CddD65qj3aqeD?Vcxh?Hnvv|14P^cA zjnLaiFY_M*&chF!9CxxSYvknkzdJ(Dc)V}fAK;WUQllu5l?RV<@Jadq&ItWpMaIVq zoOq*J3^5ss*(SyR8AqRMZBqOm9Lkml_tqSPp-K9mnRlvBmg5qze>l8+`^1I9Xc79K z@wEw7rOQTv{Q}keWBD!^nxy}k>z(~lQf4*qp1sp{EoU{^M7Bx#pW)oyGDhnHIGszr z`zeRjqJ!@L|2saf>6j=3lmW^BWq>k38K4YM1}Fo6g@LB^e-dw0e+8YK!U=Hm`Bb-< zPkV%=iHP_=LWhOVbJqL>H@WR6e!0R5E;9PxoY4R8A#YT)`E;JeT)3^cQgf9rJDN<^ z|Lu_Rkg?0B4TGB@zPBdw46&h;_WuvzjS3y_S2S=fl$EbA*mo_8g(mC&L`2)y)q~Qv z!i`hM9xJHq!qBAn|1e~`cuRKo7jQGs%C}VntDC$C{XZ1hz7cw^=nCAhPrPui8%xn4 zG%5Z+m^Z3lAp9%)J?k=B#;>^*`fm8pe13 zmQo|*R75kuns4)+mx~_iuhB9haR^qus>bm;5I`PB3-TlM8`h*9E2Rk_jI{JFM zH+IlOxQ9h!r_d%VzdKpPM5aWBCnd%wq>RwZtCH zjW}wZr~Ef%e|~fB6RG`@4JF`-kXwI)~^O#>W^LW$Nh3sBt*r zxpC}1ss7F`;b#5~npFjd|2ot|*h!fuFyAQ%dG@apsEbk;ZN6uzjxOe+0et2Y-^tyz z-bIBEb;n&)_yp>r)J2=mMYX#$+K{rYg<8XI)4z{&#T}G+hW>t^`X}|z=JQW|gGRrO z1~jId8XoK^?4`eSQ^QlJqf$q0HbNxL?Wyd&=;Zu=K070#lGV3hN{;#eEjPIi<%K&t=mh0p;(PvWb<;ExRT53kuZ z7f}4)lq=I97G1*J3A}2*5~k(?<7|O1WZ$WP-lZcNX8Cvo-`tcdU61A`)uH)G4B8Q) z-KW^9eFVM3Lrm^**jJ1k|7UDRC{-Wq1^PkT*N>ae&K4oZ|2+|!$KUk4D|A(SFfeIs zh%o0O;{Ux6x{p$~vScs_NOG7~!rJG8(4_c3Ahh$qoz=ErFreM1I};ycwn_1SX5`FW z@&ybMJ$n~Q_GF=d#s6vj|G(KYn+}IEKpCJ6PzERilmW^BWq>m9pEA(2{`cm!==7vQ zyZaF^xjG`K(%D2<9*EHY4EnJ#ue~|+c>J|g;qF!pP0atFj^OuQBPtd_w4+?0LFmU4spr>1*Xxnlj^|mm zCPnCfCxkwmvbXvZboV%8ZDy#T-~Qi(jc@kE{+7?efdJkcvgSRaU30B}I)%^L^Kw+210umtIS<1jSGoU2@~ zCjI%$oKX#aKBFebBlNE4>lf;Q{?*+}N;g?zXp;VyXVCXUEWd!R&q{|7S(Yvm5&!o@ z=)wmzx3j>&W#E`yTl`tJ$@3>^S$qBS1{RvE|B;H0*rfiW&vh_V z4h=bY3^i6BM$ud9#RcP)%W-hqVVd0BJ_X6_Mo(3(O_`%+U&R%rkHJ#{zn=- z;)LqBS%uJ5=|)E48d(fY(*M>7z2waGtA1c;CgVH7ER}^O>wlzVBX(VqNZJQRN006L zna_%%Gxh%n`Ty<+y)f+lndzV}UGWDG*!w-m`TxxI-oB_`yEaHgJNM6Wn}m&=6#r+g z*tX-nI}QSat%}?iXIPmkBJ@9VlWo14Fs(0i7u%84UW}D{AVUA!BlNn=Wf=oNe|yCn z-K%SaM~=`W{cnTN>-SeViGjhkhZi}ci&$u?|Np~hDuq!7CnD44o<@IK!pBBVep9Ivl6s*kpzD}*rVj%)c=zaeA1o1 z7JM+s>Ch`~`T0Hc$i+TGf_53?;w|4&jLaQ-56iyI@ozBfzBiirO+>mHH! zW3~(dqkO&F@3Pn%KFIo?S$kk?qgeD-rvn>wuCPW<*8dFWq1roNhJfK%55G~GY8aXn z|7R5Zq}7YWtU*83xS?_Cpg31Mc zwUrAP1wS@H{B0uWmuS>%&6dW{r2YRW$0j1dYo?Y47%W+yTWNO(LzDDBGx8C-n>RfN z{cm}Tp00brLR0d#Fhf8E-BcJ69fHuae~M{t1IxwE>G>V;5Iiydzm+jA7b1TP zdTkka`BBz%VQqpGJW2m6A@uT9Yct=1-Kj{`_ob>Bnxy}u5qjU(#go&)=E$=j_olNl zx{(gHu;eP#Pn=&;H*tOmGx0tMy(X^n{4HSph`%~1g|(|1p-KARAED1Bo6K(mRxkao zt6gA|PS*c@5jyY1+db7_HGIyUNq73Q=1t!JAA!)Pq6h8X0lnl^E`3}52t$+N|NRj< zyw*R@k;rVI%14GR3-nX=(Q z^Ts=@G(3fNSfL$Onytf1W|yuqY8Dpax$(j+RO?#$2>WS+W1Pm_w80tFOR1MOqn8>r z>ZNskP=&p;!7C>mH0OM{Rh}qH#xUcnWn?>Zr};sI>pz|A}O!MkoW60m=YnfHFWCpbSt3 zCT{OKZ>d*#|5;hTnoKS)BZ41d&RO%j{k=uboT03(F?#fy50LbPZwio(*FM# zgdTP3sQ7iTx+?3@Ye7E@P0IggY|kl=3YZPO)T+Fd@%OW$axONeK|LAlbj-AOX6RzJIi_5x`iu>~ zLgwZ|sVarPQdK4-Tn;P;TlECnS8EqyXj1&2Nt2oR<7 z)0TLrL$B9`@?s~uV`y`0u0(^8^93X4Gb3kwZFuD6Sqfky>$yqYCj{U-2-OqT! z_|;&e*8f|-5Y~EHZQ=T0XgfQ+r!BiA zJ9+=V4?=sp#p!v1;Rk0ks~;Z1p-EBk{|p50>*KQ;)&DC9E!hy;7lV`H|MG&N>w-bx z3A;eaSs0oW|93#>m~NHd-N4{v++M{MS6OHj7bV31dn2^jl`aayz(DP;3XDRBp8JqfjG@gexh)$+lWQ2iLUHgK zCYTHe9rWkbGI`K{IWl1219k+T-2cxLp?M&Fqy`L^@a+5cy1=qcj{iF$bcon@lMc{z z%=$rk7wj-JY5zZS#Vm8mU)P|zhZPgn4^G0+r1-xpLOb<|?qCMp%v)dUk^KZio0)Q3 zG+=F}fVG(nYjz0jCYRr?2N+~#`TBdYx+aQ<|GOh}=;c34JfZ9J9_fGXcqVL{+5b<7 z|12VW|S`|My>eJkvi+8K4YM1}FoR0m=YnfHFWCXjTTA z*8fPzZgvYSeR>ECKffJo+jX+AxDb*5kHp1hC%d#7P0*`)7PEFSyA_>?{QoorcQH9^ zZUu&GgS|_v&SS<&`dh$I_yx_8>|BGnXtx=9T*>Fr348e$amjW}B4%Z;8<65j*-tg2D2#uGKbWEHqjF zGq(NDnLj88-7^Q2EKknH&?Noe3!yE>Y+IuUT{oK-{gJ%~LzDDB(%_rDr)|8D3uUKI>qM1|!hzGtC-<^QY7KP2n_0SY?k6#AkJ zPzERilmW^BWq>k38K4YM2LAR8G`#?TqKjKDCtaO70t^FXb!YTf5Y`7G764FObj#Mn znhYb*Uwb&XqQf@~%`xXH*GEFz9WzWb6e+%r;5?yCK_$_IgDg0zIo+5sqOT7MiU8k&@lAi<=!X3-r9_AL6fJ zcP=FF|3?x0Eqg7wS~g%{vvlCS=Z7)dB>j(c@RoG}Z_4t((Dlam*?ZZGE@b`BMd(wS zF@xrT;gk(FgFY?AY?JiAJwh)&ZL;7P7`*WMW|tJqLX-7BGw;(wXSt|?QPk{xTO`FW zG)ez^AoPNn-#Ti6!IjpAONUIt&?NnjR!z1XA9Bt?84NYHpX($RiJ?jQ-wUDt9P)L@ z12EXUE-T9XhcH@%{%5XO(euL_euCk&IY+Kc?uemD@qeV?x9p2p@p3Qd_2uoTni`Fv zN&26M(3{^Wo4f}7t|MKg48vIHTK%usM@(AGPE2l_Y#XVu689Cu&_66hJk$`Tt_Hgf zM@Egq5zmclthBY0Q;;L87%OwguJH<&b){ID6KH))THmtS>RV>2naZg3=pmjPDeR(k zT>|*O{ZsJoUX44a;9csO)H9pWGff)x%(@3*y=MykZQHnK3f`rjNj^Cf zy52Jd|F&t|GX?Kb&!nE&e4eRq+-O<#y3c1;S@nhots3{yhNnt(kaT|Bl`V*+5Qb%nzN3GlcPvie0Ry^q2lmW^BWq>k38K4YM1}FoR zf&UQ(nvVZF9(ks6b=}jhXm4`C{A-oK@MZk@x>I=+z$Bqv|C`*?M|x$+Hsg|Qv9E}Vr6L+ z5C#T^O=s+?vBc1%`2VlF!~^sXZMV&zks^#15&ySCw(mJ)k1+)Ogs1bDUHgflN%4Pl zy;Tl9Q|iAT4ATzZt(?xq(4_dk{gG#Ky&V#cs(=wstkbdhvn+Id{y*LS|3C6Dr^BQS zPzERilmW^BWq>k38K4Z18E9Jm_dfFU`~CT$@~=Sui+YczV&%f}K!p5vLFk8J^YnUw zZphx<>C!3~n%Mtu3W7iIF`JSKdY)B#&!ywd2Z{Or2%U2#YluGRRZ8#Oy(k1TPSXDl z2wnR0)lOs3Pk;OIbhr%*P1gUe2)$?Ep|dHV$F*>{Fnu$IHn-%;H`w=3+g*)U?XOZ> zgAbvr{pJ^U0DZe{JF4~5F*Hg4dm{7|^ONO+q06V2-N*Fv!_XxCZ-UUb73Q{&0lhSr z!)q7p#L%Ste;z{rv<`Vx00uEvZeF)&!9tVuKljMfA7VC(6<>gXS;C-a-mHD_9J2m* zMrh^5>-WqBy=8scU)eewvrW?f=4j-fhgyyq2fEzcs=eI~VrY{7w?ejcB3JZrgD%H% zIXN#``yNE-e}?lfW*RDnpqDdBCbNQ7u~EeSe=meCiGS460Sw)xZh>Zn@W>IGr2m}| zy3l9I=doZIkrnZ*pEwI$um5TMpZpM|?@|UR1C#;E0A+wOKpCJ6PzL^27-)I{!1~BD z$+9&wRcC`y|B;tZy}2%|4@4{g^g8maMN0!?Wg9SD`Qf(D@*oUNivLeUwy$2=?dAYR zJABl{%uO&fN&lnxzvPw=mmhn8p{8;E!g*{ZTdRu{8)$G7=4v4gZo+8r-e}})OBWWN z1O0yIyxdJSS+>dgAH`%PE4t_mn+|#(+l(I(%Bre{YVeWtKM&cK{&K2C5Ew+?w%NUM zK4zO^$(5@gIlr)O3?*^B%ciJw=ffoRQn`F zUchS)lJvh1LSG3^xsm{edq#P`liR?uP1gTt-jWX-#|P(w-t}l>*@9~rnzaAl6|tsi zvDCp449YztZr=HS?VWdAQ&<1T0}4SD6@3)jI#2`xxDXKsauRlA1q>ph1f0c*=tJus zEh>T*sC2 zs+~d;A|Dc2{-FQ6zq$#d=GFg$s5&}!_@#}+DO*{0%Ljc#Y9sldsyl|5bs)sWW%=b5 zo7nCF-26|~5wq^M`H;vq?Wk3I!x3g|UjFY+)vk`SCOjf;7oR7cw-wL9=db@4Ox59c zf-ikeoYzlm-s`BA8Jm~?1F5>*Blq2Hh-}!`Gxw%uiPZf3??cs5%MN_zNZe+h+3|Cn zD2p1Y|NBz4^_|S?TZrSTrfqGr)Xdnt{NItP+n9aQMoFAI9_$l&eJ`Wt=6|a8-ul%E zHIe#;b=li15$T?eyY)_vkzgnUxI_Uq$y>Q||5C8!X009sH0T2KI5U7F#D$f7@dn7kg zM>eXNPn@rJ{%OL+K*MNaB>&T3b90jWwStR8*1=-^OP`vInmhkLlgd91KN_1voVU$= z*l|D@Bj@G+T2wu1vvk;e;-F8j8@$w?QSF7GWNX^gxg8Ji) zKR1vPXYJ`dlWU1P+41ZDAyhpnYw*eK#3g-V-xKjk%-FpAub}GqjGWb}#HDtVTCP3B zJ^1+fUpQrB!yj*(OJwu!5mj}C)m_)q#Km{s zoC$rC88t8e3u6ydJh{4wv~JlT^PWOHgP%YDpMGmn_$v9F$dZ0<-~Fkm=R%qMkNN*q z@XC%u1_2NN0T2KI5C8!X0D*D{RQv^ikjdKY37>MAxG3*AoNdwAFq;Tp0C;-WEtBZ# z3V)s^y23lz+fudrx#gqh5Lx=u?9eMS88vVIe=1cw+;7`;8F7((HQUxLic$0Ozb#ca z&cFT7Od|U|rNxQ;Cm1y^|I4Ymm3QvFZ;4CG^M~5a_(-JY=YJthx`$_s-bvg%UEaGE zw~|rw>iMm=p zd^DF)^XC6|p=#Td8so1Lr}@D-*Lp2v)V%yptLQg7S~aN|M`V9&nwPAZ%BXqyU--^N z*6L?lSK@YZNJzwB7elp?{4bpEPQ9zUeotI`e)C%|s}M%b%l~0i?eg-ct;dP1^`$}0 zSBUx^8ma#a&%3F%oqQ9Kk=#0c>xvfmD9Qhpl0b=dzLkT;0`rHowO^pbr<}`3MVpEm z0XlUy=*tqOjMC8FDn0xHG>UMAMja5Qa#xnO(ciGyMR~iR-38iRn5^AJnv)k}RN7gE zHL7@{FKblsE-)&LYC5A@b(GpUIGCl!6!xv`q0|KR4pXVyD?-ZrsxUBRnFnA+SkZJ= ztnDeab9OdMk1J}SD!fUn{G$us)eGO$i~l1xu%p34FeD6VIz!rbD7URzQ8zz>wrZuD zd<@MQmb=UHTltBn=dkl&a3~c>DZZ?&ky`bR!wTS=Dc>lNX@VRN7E-h+gy6)7+JdPNww-W zqFL2O*8hu?r-V1?UyzzZRy<$$VQzLBGd8dOFO2=^jA!3xkd@Jk%x^lfYY+13|8aC| zn;{eX{6dzh_xUudahDmJSO4!%)%_=!xg?RLPoFil+WnzO&9DFWrRrK_-r{Ox+4^UT zexKZlQS<8my{Nic_WfJ#Wa$@MN8Q}|X6ygO()0a?dn@%dvT|Ab)P2f%%-Fp8zlx6C zb-ZkTFj;nH*4B(U`wZ1a>i_h}!)~9FJ1rzDk9=2uTGL~UnpgktNzeC|YIXdiWcl9( z2S#VSS^q!k7PZg!3lkfW6`8eXX4g&;jZJ;-A>@Ba;k=$IbydT`f&d7B00@8p2!H?x zfB*=900@8p2)qG-qWVAX|9=By*cAjo00ck)1V8`;KmY_PF@cIF00hcY;=&%C{`D}) zy^(n-Wb!S;`k;{s0Dkh6cn@Fq;VnsS?DWYWDa6%eVFG}6ozm3n%`oElx?cVc}hYMM9Vz)#){Ch^ttN(YEr}PQzWN~~V z(cL|}ZT%rjk(yus?uPTt@zM$N1L3rGGmJnzOUq8-;LGXGS7q1s6OzY|r@vb7xVN^+-N zc{H&J+dY6+|EJ93>x4`nU{3TiY*s3Nc*TrOeQs9z8jEaK^cst7C%ne!k@s^uG<)Q~ zh;G<}UFnS)iPXjQf7_bK|CRUxjza?h5C8!X009sH0T2LzHy}`P{tuF;Xhyu6Q|}j& zGwYDe&sTOC<^v=7UqRD_SL0@(mL&I9P{fmo;(l=a{GUq49UA!PD$GSR zMQc9Xcr$?}x}lvWjj?+!8k?X0edu=&8R`u?Pm^4m&e|_mi)XU)*Z=pYV^8({afKDp zwHWYn%7ZD)*u4Btb8zB$dG51hq6^c7Z|IrHsCoIHsx{r$$b$WeJ|wo)__kJznwS6S z^G+BW^mO4VqIbR(=o8jmq~_;;VeB~fxoTUI`>pxX_ah!KYF_@QUem;_oib`Q$@$!I zhh)J>M$OCrf;zQkLGU@UxXr;y&iC#yYF_@Q-;yQx`QCc*1Ieuy(K0*P*HCRF|I-Ao z*|t{K^C{6T-*})wzNk|rRU66w{Tjv8YPOjyj_NIaoFF=Ks^;bYj`Ea2_P!I2oF`hx z)d@3WVnjz?n*VYC{~KITu`39G00@8p2!H?xfB*{flm;bTNQ)^VSW zLR`@0FaIyZqF-xWy%J3H<3BcU=prg?(k~A@Ji8T^|IMXR525TYFe)zpOFp54$97ct zJR;f;hNe`L(g zY;o^Fe*UKkK4F^KfpMdVw&i!-lk>#)qVw}Vtp~?zCbty@Q{Z|LN?3 zgn2{T1~ex6v8qk`&vg+!Z+`zj;e<;DW<9w;bg>aZeJ1o`)V%y3N|S7x=4VGNCi-hn z0&mADMQVQjr|S4-11vkAB>E@yzZ+NQA*1Hye>&$PzS$|aeZz@9uEn4mR`HCQm;WiX z_@0Rt^(=|5-8{!8f!U0jm;bv^_00)meyByXaczbTiTJ`$Z6yB-%*QSZ)$b+x0BPg* z4vW6IFp~f2Hx~&HJ`3EXBRR)vD7GpZFk|!bKXvMWubx_;)l>C}cF|)Siqy#emG`oZ zBLe{t009sH0T2KI5CDO?UWT z4L&Qcv9+dQxGW|TPaiw29e}-hF&2{&Yzpry8a$U&CCA+wZy

`Npmlo@T5LQqdl8DO_qomv&)XTKwK!_F5|17Qnhxvx+d^2ciuQn4;Z>7)HO;|?acS{8cv+W%DWC=APkMsEM7>i)e)`l>q< zopVBu(D>zynmhkLlMX)T`KrFViSFx&O)YEOVdT8}e@!~G+3RZi$KFJ%xTtB=cO9eV z)&J>??u7I`x3%+$cCp!*$w@&XHGlp;olV{^zaU`AN}?T*+3fHyH5fH-{XYSEmfN4c z_C){C-6c@8hm0({?V&bNv^fo3D=9F)#PXm-m|^8v~CoQWvQ(`~E>f$K1vOo{Fk#5Fd5dD{9Q-azj)y)H9XY@LFFfzH zU;pj+iY)P%{zuDas~9z}{!eEQBu34RaL*)*pMMhUc5%3&+DQFhxJ~9vYE<2IM7t^P z#-M|)jG9;f7hYsu=`zQ$L~H$Q;q+0WyG7{Oy!t<-Hn>Io^OGCUdk=Q)eAqcbGf=TkTAiLT&9z8PT?JTL!S(RuLp-P`{7k}P)% zelRqrB{MiL|I-v5(`@XR|8ylfvzITnY}hYS(_GZayH=Sj`Z~#~3X^OYec}oII%F;; z+Vx8|p1Umm=7K-}pU&ux@g3z~=Wn8q8o%z%xC+=1wKet#?8L%iN2KmQBVEb|&Y3MIMcbgw+eh~2`^|1?F% zbhC>u$RzrCdCh`UIn0ss^1txBFMK)uke1}M%G)q=&MZTVq#agV{H1oCJDa5sFYW&q9HI$|aqpl}MfA`_ zDb&gcWtcKb!)%RY`&Tggpo0ZESeUGXMOqVIshzvKS^7W&uLj0w{Gy`-w-)=fbW23J z9xZ(w{)9hG=T8R((+7xN5vkjlD{0wn;T3tbLI+acHO#}rH?B5EGoJd7UrjN zuA{}s009sH0T2KI5C8!X009sH0aFt|{x>yI z{5=o=0T2KI5C8!X009sH0T2KI5U6|vkpC;+Il&Qu00@8p2!H?xfB*=900@8p2!Md8 z2_XNQnkfDr2!H?xfB*=900@8p2!H?xfB*VR^FJu2y@$9fMcHjV4UJmTU!_z9m%^o{ zbE!*^!J7YN{Z`UtqPTlXp-)S;Mpx|Al6~M+c-3@XZ5zm6S)E=Z%UxN$XuI0bugtpa zMSH=qaIEPZ>kv@>X8_tmP1KpHaAp73+oBAeT5?S;Rl%ijsmWZ5`v2P}Qz$_I1V8`; zKmY_l00ck)1V8`;K%lA+K>n{PX9~v&0w4eaAOHd&00JNY0w4eaAOHeyPXPJGPlr}Z>L=V@+Hmars5y# zb>@qTf5*QdP+#x-8heU79aou zAOHd&00JNY0w4eaAOHd&z$1YC&!dCKKmY_l00ck)1V8`;KmY_l00cnb-5`Mc|8AT| zJPQy20T2KI5C8!X009sH0T2KI5a1C&{^!xbV;}$mAOHd&00JNY0w4eaAOHd&@NN)5 z{(m>lBc25afB*=900@8p2!H?xfB*=900{612>IW_{D6dZ{@Y_^XEB2QiXRZDS_IOZ zq6}JC1^KHqO0`C4w${T>(?hLLNB2;cx3QmLvz798LAMojTQOO;l{BZwii{c>Q;tzf zHYQeR)RJ9bR2bESMpad0RAqTr)RK*w3XNK_3ycb*n$Dt1" over a defined period of -time is needed, the module looks for all elements where the condition applies -and where it does not apply. This generates two lists, which contain the "start" -and "end" times of the condition. -A futher function combines the start- and endtimes to timetouples between which -the condition applies. A "state" function returns True/False for an exact time +"""Module for generating conditions for mnemonics + +The modules purpose is to return True/False for several times by reference of +certain conditions. If for instance the condition "x>1" over a defined period of +time is needed, the module looks for all elements where the condition applies +and where it does not apply. This generates two lists, which contain the "start" +and "end" times of the condition. +A futher function combines the start- and endtimes to timetouples between which +the condition applies. A "state" function returns True/False for an exact time attribute, whereby the condition is represented in binary form. Authors @@ -16,35 +16,35 @@ Use --- - This module is not prepared for standalone use. + This module is not prepared for standalone use. - For use in programm set condition up like below: + For use in programm set condition up like below: - import the module as follow: + import the module as follow: >>>import condition as cond - generate list with required conditions: - >>>con_set = [ cond.equal(m.mnemonic('IMIR_HK_POM_LOOP'),'OFF'), - cond.smaller(m.mnemonic('IMIR_HK_ICE_SEC_VOLT1'),1), + generate list with required conditions: + >>>con_set = [ cond.equal(m.mnemonic('IMIR_HK_POM_LOOP'),'OFF'), + cond.smaller(m.mnemonic('IMIR_HK_ICE_SEC_VOLT1'),1), cond.greater(m.mnemonic('SE_ZIMIRICEA'),0.2)] - generate object of condition with the con_set as attribute: + generate object of condition with the con_set as attribute: >>>condition_object=cond.condition(con_set) - Now the condition_object can return a True/False statement wheather - the time given as attribut meets the conditions: + Now the condition_object can return a True/False statement wheather + the time given as attribut meets the conditions: >>>if condition.state(float(element['Primary Time'])): - -> True when condition for the given time applies + -> True when condition for the given time applies -> False when condition for the given time is not applicable Dependencies ------------ - no external files needed + no external files needed References ---------- - + Notes ----- @@ -54,36 +54,36 @@ class condition: """Class to hold several subconditions""" - #contains a list of all necessary time pairs + #contains list of representative time pairs for each subcondition cond_time_pairs = [] #state of the condition __state = False - #initializes condition through condition set + #initializes condition through condition set def __init__(self, cond_set): - """Initialize object with set of conditions + """Initialize object with set of conditions Parameters ---------- - cond_set : list + cond_set : list list contains subconditions objects """ self.cond_set = cond_set - #destructor -> take care that all time_pairs are deleted! + #destructor -> take care that all time_pairs are deleted! def __del__(self): """Delete object - destructor method""" del self.cond_time_pairs[:] #prints all stored time pairs (for developement only) - def print_times(self): + def print_times(self): """Print conditions time pairs on command line (developement)""" print('Available time pairs:') - for times in self.cond_time_pairs: + for times in self.cond_time_pairs: print('list: '+str(times)) - + #returns a interval if time is anywhere in between def get_interval(self, time): - """Returns time interval if availlable, where "time" is in between + """Returns time interval if availlable, where "time" is in between Parameters ---------- time : float @@ -93,7 +93,6 @@ def get_interval(self, time): time_pair : tuple pair of start_time and end_time where time is in between """ - end_time = 10000000 start_time = 0 @@ -106,28 +105,28 @@ def get_interval(self, time): start_time = pair[0] end_time = pair[1] break - else: + else: break if (end_time != 10000000) and (start_time != 0): return (start_time, end_time) - else: - return None + else: + return None #generates time pairs out of start and end times def generate_time_pairs(start_times, end_times): - """Forms time pairs out of start times and end times + """Forms time pairs out of start times and end times Parameters ---------- - start_times : list - contains all times where a condition applies - end_times : list - contains all times where the condition does not apply + start_times : list + contains all times where a condition applies + end_times : list + contains all times where the condition does not apply Return ------ - time_pair : list - list of touples with start and end time + time_pair : list + list of touples with start and end time """ #internal use only time_pair: float = [] @@ -137,10 +136,10 @@ def generate_time_pairs(start_times, end_times): time_pair.append((0,0)) #check if the condition indicates an open time range - elif not end_times: + elif not end_times: time_pair.append((start_times[0], 0)) - #generate time pairs + #generate time pairs #for each start time a higher or equal end time is searched for #these times form am touple which is appended to time_pair : list else: @@ -149,13 +148,13 @@ def generate_time_pairs(start_times, end_times): for start in list(sorted(set(start_times))): - if(start > time_hook): + if(start > time_hook): for end in list(sorted(set(end_times))): if end > start: time_pair.append((start, end)) - time_hook = end + time_hook = end break if list(sorted(set(start_times)))[-1] > list(sorted(set(end_times)))[-1]: @@ -165,52 +164,52 @@ def generate_time_pairs(start_times, end_times): #returns state of the condition at a given time - #if state(given time)==True -> condition is true + #if state(given time)==True -> condition is true #if state(given time)==False -> condition is false def state(self, time): """Checks whether condition is true of false at a given time Parameters ---------- - time : float + time : float input time for condition query Return ------ - state : bool - True/False statement whether the condition applies or not + state : bool + True/False statement whether the condition applies or not """ #checks condition for every sub condition in condition set (subconditions) - state = self.__state + state = self.__state - for cond in self.cond_time_pairs: + for cond in self.cond_time_pairs: if self.__check_subcondition(cond, time): state = True - else: - state = False + else: + state = False break - - return state - - def __check_subcondition(self, cond, time): + return state + + + def __check_subcondition(self, cond, time): #if there are no values availlable if cond[0][0] == 0: return False - + for time_pair in cond: - #if just a positive time is availlable, return true + #if just a positive time is availlable, return true if (time_pair[1] == 0) and (time > time_pair[0]): - + return True - #if given time occurs between a time pair, return true + #if given time occurs between a time pair, return true elif (time_pair[0]) <= time and (time < time_pair[1]): - - return True - else: + return True + + else: pass @@ -219,29 +218,28 @@ class equal(condition): #add attributes to function - start function "cond_time_pairs()" def __init__(self, mnemonic, value): - """Initializes subconditon + """Initializes subconditon Parameters ---------- mnemonic : astropy table - includes mnemomic engineering data and corresponding primary time + includes mnemomic engineering data and corresponding primary time value : str - coparison value for equal statement + coparison value for equal statement """ self.mnemonic = mnemonic self.value = value - condition.cond_time_pairs.append((self.cond_true_time())) - - #generates a list of time-touples (start_time, end_time) that mark the beginning and end of + + #generates a list of time-touples (start_time, end_time) that mark the beginning and end of #wheather the condition is true or not def cond_true_time(self): - """Filters all values that are equal to a given comparison value - if equal: Primary time -> temp_start - if not equal: Primary time -> temp_end + """Filters all values that are equal to a given comparison value + if equal: Primary time -> temp_start + if not equal: Primary time -> temp_end Return ------ - time_p : list + time_p : list list of touples with start and end time """ temp_start = [] @@ -253,12 +251,11 @@ def cond_true_time(self): if key['value'] == self.value: temp_start.append(key["time"]) - #find all end values + #find all end values else: temp_end.append(key["time"]) time_p = condition.generate_time_pairs(temp_start, temp_end) - return time_p @@ -267,28 +264,27 @@ class greater(condition): #add attributes to function - start function "cond_time_pairs()" def __init__(self, mnemonic, value): - """Initializes subconditon + """Initializes subconditon Parameters ---------- mnemonic : astropy table - includes mnemomic engineering data and corresponding primary time + includes mnemomic engineering data and corresponding primary time value : str - coparison value for equal statement + coparison value for equal statement """ self.mnemonic=mnemonic self.value=value - condition.cond_time_pairs.append((self.cond_true_time())) def cond_true_time(self): - """Filters all values that are greater than a given comparison value - if equal: Primary time -> temp_start - if not equal: Primary time -> temp_end + """Filters all values that are greater than a given comparison value + if equal: Primary time -> temp_start + if not equal: Primary time -> temp_end Return ------ - time_p : list - list of touples with start and end time - """ + time_p : list + list of touples with start and end time + """ temp_start: float = [] temp_end: float = [] @@ -298,55 +294,56 @@ def cond_true_time(self): if float(key['value']) > self.value: temp_start.append(key["time"]) - #find all end values + #find all end values else: - temp_end.append(key["time"]) + temp_end.append(key["time"]) + + time_p = condition.generate_time_pairs(temp_start, temp_end) + return time_p - return condition.generate_time_pairs(temp_start, temp_end) class smaller(condition): """Class to hold single "greater than" subcondition""" #add attributes to function - start function "cond_time_pairs()" def __init__(self, mnemonic, value): - """Initializes subconditon + """Initializes subconditon Parameters ---------- mnemonic : astropy table - includes mnemomic engineering data and corresponding primary time + includes mnemomic engineering data and corresponding primary time value : str - coparison value for equal statement + coparison value for equal statement """ self.mnemonic=mnemonic self.value=value - condition.cond_time_pairs.append((self.cond_true_time())) def cond_true_time(self): - """Filters all values that are greater than a given comparison value - if equal: Primary time -> temp_start - if not equal: Primary time -> temp_end + """Filters all values that are greater than a given comparison value + if equal: Primary time -> temp_start + if not equal: Primary time -> temp_end Return ------ - time_p : list - list of touples with start and end time - """ + time_p : list + list of touples with start and end time + """ temp_start: float = [] temp_end: float = [] - + for key in self.mnemonic: #find all times whose Raw values are grater than the given value if float(key['value']) < self.value: temp_start.append(key["time"]) - #find all end values + #find all end values else: temp_end.append(key["time"]) - return condition.generate_time_pairs(temp_start, temp_end) + time_p = condition.generate_time_pairs(temp_start, temp_end) + return time_p -if __name__ =='__main__': +if __name__ =='__main__': pass - diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/csv_to_AstropyTable.py b/jwql/instrument_monitors/miri_monitors/data_trending/csv_to_AstropyTable.py index 104078839..c96118292 100755 --- a/jwql/instrument_monitors/miri_monitors/data_trending/csv_to_AstropyTable.py +++ b/jwql/instrument_monitors/miri_monitors/data_trending/csv_to_AstropyTable.py @@ -8,7 +8,7 @@ Authors ------- - - Daniel Kühbacher + - Daniel Kühbacher Use --- @@ -18,7 +18,7 @@ ------------ mnemonics.py -> includes a list of mnemonics to be evaluated - + References ---------- @@ -27,20 +27,17 @@ ----- """ -from astropy.table import Table +from astropy.table import Table from astropy.time import Time +import warnings import mnemonics as mn class mnemonics: - """class to hold a set of mnemonics""" + """class to hold a set of mnemonics""" __mnemonic_dict = {} - __mnemonic_label = "Telemetry Mnemonic" - __time_label = "Secondary Time" - __value_label = "EU Value" - def __init__(self, import_path): """main function of this class Parameters @@ -48,7 +45,6 @@ def __init__(self, import_path): import_path : str defines file to import (csv sheet) """ - imported_data = self.import_CSV(import_path) length = len(imported_data) @@ -58,7 +54,10 @@ def __init__(self, import_path): for mnemonic_name in mn.mnemonic_set_base: temp = self.sort_mnemonic(mnemonic_name, imported_data) #append temp to dict with related mnemonic - self.__mnemonic_dict.update({mnemonic_name:temp}) + if temp != None: + self.__mnemonic_dict.update({mnemonic_name:temp}) + else: + warnings.warn("fatal error") def import_CSV(self, path): @@ -72,11 +71,10 @@ def import_CSV(self, path): imported_data : AstropyTable container for imported data """ - #read data from given *CSV file imported_data=Table.read(path, format='ascii.basic', delimiter=',') return imported_data - + #returns table of single mnemonic def mnemonic(self, name): @@ -92,12 +90,12 @@ def mnemonic(self, name): """ try: return self.__mnemonic_dict[name] - except KeyError: + except KeyError: print('{} not in list'.format(name)) #looks for given mnemonic in given table - #returns list containing astropy tables with sorted mnemonics and engineering values + #returns list containing astropy tables with sorted mnemonics and engineering values #adds useful meta data to Table def sort_mnemonic(self, mnemonic, table): """Looks for all values in table with identifier "mnemonic" @@ -106,7 +104,7 @@ def sort_mnemonic(self, mnemonic, table): ---------- mnemonic : str identifies which mnemonic to look for - table : AstropyTable + table : AstropyTable table that stores mnemonics and data Return ------ @@ -114,22 +112,25 @@ def sort_mnemonic(self, mnemonic, table): stores all data associated with identifier "mnemonic" """ - temp1: float = [] + temp1: float = [] temp2 = [] #appends present mnemonic data to temp arrays temp1 and temp2 for item in table: - if item[self.__mnemonic_label] == mnemonic: - #convert time string to mjd format - temp = item[self.__time_label].replace('/','-').replace(' ','T') - t = Time(temp, format='isot') - - temp1.append(t.mjd) - temp2.append(item[self.__value_label]) - + try: + if item['Telemetry Mnemonic'] == mnemonic: + #convert time string to mjd format + temp = item['Secondary Time'].replace('/','-').replace(' ','T') + t = Time(temp, format='isot') + + temp1.append(t.mjd) + temp2.append(item['EU Value']) + except KeyError: + warnings.warn("{} is not in mnemonic table".format(mnemonic)) + description = ('time','value') data = [temp1, temp2] - + #add some meta data if len(temp1) > 0: date_start = temp1[0] @@ -142,9 +143,10 @@ def sort_mnemonic(self, mnemonic, table): info['mnemonic'] = mnemonic info['len'] = len(temp1) - #table to return - mnemonic_table = Table(data, names = description, dtype = ('f8','str'), meta = info) + #table to return + mnemonic_table = Table(data, names = description, \ + dtype = ('f8','str'), meta = info) return mnemonic_table -if __name__ =='__main__': - pass \ No newline at end of file +if __name__ =='__main__': + pass diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/data_extract.py b/jwql/instrument_monitors/miri_monitors/data_trending/data_extract.py index 444b26e7f..aa027b757 100755 --- a/jwql/instrument_monitors/miri_monitors/data_trending/data_extract.py +++ b/jwql/instrument_monitors/miri_monitors/data_trending/data_extract.py @@ -1,4 +1,4 @@ -"""Functions that are used to extract data +"""Functions that are used to extract data ------- @@ -10,11 +10,11 @@ Dependencies ------------ - no external files needed + no external files needed References ---------- - + Notes ----- @@ -24,21 +24,23 @@ import condition as cond import statistics import sqlite3 +import warnings from collections import defaultdict + def extract_data(condition, mnemonic): - '''Function extracts data from given mnemmonic that applies to the given condition + '''Function extracts data from given mnemmonic that applies to the given condition Parameters ---------- - condition : object + condition : object conditon object that holds one or more subconditions - mnemonic : AstropyTable + mnemonic : AstropyTable holds single table with mnemonic data Return ------ temp : list or None - holds data that applies to given condition + holds data that applies to given condition ''' temp = [] @@ -48,57 +50,60 @@ def extract_data(condition, mnemonic): #developement purpose: #print("condition true: value= {}, time= {}".format(element['value'],element['time'])) temp.append(float(element['value'])) - - if len(temp) > 0: + + if len(temp) > 0: return temp - else: + else: return None def extract_filterpos(condition, nominals, ratio_mnem, pos_mnem): - '''Extracts to filterpositions corresponding ratio values + '''Extracts to filterpositions corresponding ratio values Parameters ---------- - condition : object + condition : object conditon object that holds one or more subconditions - ratio_mem : AstropyTable - holds ratio values of one specific mnemonic + ratio_mem : AstropyTable + holds ratio values of one specific mnemonic pos_mem : AstropyTable holds pos values of one specific mnemonic Return ------ - pos_values : dict + pos_values : dict holds ratio values and times with corresponding pos label ''' - + pos_values = defaultdict(list) #do for every position in mnemonic for pos in pos_mnem: - interval = condition.get_interval(pos['time']) + if pos['value'] != "UNKNOWN": - if interval is not None: - cur_pos_time = pos['time'] - filtername = pos['value'] + interval = condition.get_interval(pos['time']) - for ratio in ratio_mnem: + if interval is not None: + cur_pos_time = pos['time'] + filtername = pos['value'] - if (ratio['time'] >= cur_pos_time) and \ - (abs(float(ratio['value']) - nominals.get(pos['value'])) < 10): + for ratio in ratio_mnem: - if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): - pos_values[pos['value']].append(( ratio['time'], ratio['value'])) - break - + if (ratio['time'] >= cur_pos_time) and \ + (abs(float(ratio['value']) - nominals.get(pos['value'])) < 10): + + if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): + pos_values[pos['value']].append(( ratio['time'], ratio['value'])) + break + else: + warnings.warn("Position for {} is UNKNOWN".format(pos['value'])) return pos_values -def once_a_day_routine(mnemonic_data): +def once_a_day_routine(mnemonic_data): - #abbreviate attribute + #abbreviate attribute m = mnemonic_data - ######################################################################### + ######################################################################### con_set_1 = [ \ cond.equal(m.mnemonic('IMIR_HK_IMG_CAL_LOOP'),'OFF'), \ cond.equal(m.mnemonic('IMIR_HK_IFU_CAL_LOOP'),'OFF'), \ @@ -115,7 +120,7 @@ def once_a_day_routine(mnemonic_data): if data != None: data_cond_1.update( {identifier:data} ) - else: + else: print("no data for {}".format(identifier)) del condition_1 @@ -136,20 +141,20 @@ def once_a_day_routine(mnemonic_data): if data != None: data_cond_2.update( {identifier:data} ) - else: + else: print("no data for {}".format(identifier)) del condition_2 - return data_cond_1, data_cond_2 + return data_cond_1, data_cond_2 #in developement -def whole_day_routine(mnemonic_data): +def whole_day_routine(mnemonic_data): - #abbreviate attribute + #abbreviate attribute m = mnemonic_data - ######################################################################### + ######################################################################### con_set_3 = [ \ cond.greater(m.mnemonic('IMIR_HK_ICE_SEC_VOLT1'), 25.0)] #setup condition @@ -162,7 +167,7 @@ def whole_day_routine(mnemonic_data): if data != None: data_cond_3.update({identifier:data}) - else: + else: print("no data for {}".format(identifier)) del condition_3 @@ -175,7 +180,7 @@ def whole_day_routine(mnemonic_data): FW_volt = extract_data(condition_FW, m.mnemonic('IMIR_HK_FW_POS_VOLT')) del condition_FW - + con_set_GW14 = [ \ cond.greater(m.mnemonic('IMIR_HK_GW14_POS_VOLT'),250.0)] #setup condition @@ -199,16 +204,16 @@ def whole_day_routine(mnemonic_data): CCC_volt = extract_data(condition_CCC, m.mnemonic('IMIR_HK_CCC_POS_VOLT')) del condition_CCC - + return data_cond_3, FW_volt , GW14_volt, GW23_volt, CCC_volt - -def wheelpos_routine(mnemonic_data): - #abbreviate attribute +def wheelpos_routine(mnemonic_data): + + #abbreviate attribute m = mnemonic_data - + con_set_FW = [ \ cond.greater(m.mnemonic('IMIR_HK_FW_POS_VOLT'),250.0)] #setup condition @@ -230,36 +235,36 @@ def wheelpos_routine(mnemonic_data): con_set_GW23 = [ \ cond.greater(m.mnemonic('IMIR_HK_FW_POS_VOLT'),250.0)] #setup condition - condition_GW23 = cond.condition(con_set_FW) + condition_GW23 = cond.condition(con_set_GW23) GW23 = extract_filterpos(condition_GW23, mn.gw23_nominals, \ m.mnemonic('IMIR_HK_GW23_POS_RATIO'), m.mnemonic('IMIR_HK_GW23_CUR_POS')) del condition_GW23 - + con_set_CCC = [ \ cond.greater(m.mnemonic('IMIR_HK_CCC_POS_VOLT'),250.0)] #setup condition - condition_CCC = cond.condition(con_set_FW) - CCC = extract_filterpos(condition_CCC, mn.fw_nominals, \ + condition_CCC = cond.condition(con_set_CCC) + CCC = extract_filterpos(condition_CCC, mn.ccc_nominals, \ m.mnemonic('IMIR_HK_CCC_POS_RATIO'), m.mnemonic('IMIR_HK_CCC_CUR_POS')) - + del condition_CCC return FW, GW14, GW23, CCC def extract_filterpos_obsolete(condition, ratio_mnem, pos_mnem): - '''Extracts to filterpositions corresponding ratio values + '''Extracts to filterpositions corresponding ratio values Parameters ---------- - condition : object + condition : object conditon object that holds one or more subconditions - ratio_mem : AstropyTable - holds ratio values of one specific mnemonic + ratio_mem : AstropyTable + holds ratio values of one specific mnemonic pos_mem : AstropyTable holds pos values of one specific mnemonic Return ------ - pos_values : dict + pos_values : dict holds ratio values and times with corresponding pos label ''' @@ -274,14 +279,14 @@ def extract_filterpos_obsolete(condition, ratio_mnem, pos_mnem): cur_pos_time = pos['time'] filtername = pos['value'] - for ratio in ratio_mnem: + for ratio in ratio_mnem: - if (ratio['time'] == cur_pos_time) and (abs(float(ratio['value'])) > 0.1): - if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): + if (ratio['time'] == cur_pos_time) and (abs(float(ratio['value'])) > 0.1): + if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): pos_values[pos['value']].append( ( ratio['time'], ratio['value']) ) break return pos_values -if __name__ =='__main__': +if __name__ =='__main__': pass diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py b/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py index 6fa29e7b3..a65c9ee6f 100755 --- a/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py +++ b/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py @@ -14,55 +14,63 @@ #create filename string directory='/home/daniel/STScI/trainigData/set_1_day/' -""" -filenames = [ + +filenames1 = [ 'imir_190204_DoY2292017FOFTLM2019035182609402.CSV', -'imir_190204_DoY2402017FOFTLM2019035180907145.CSV', -'imir_190204_DoY2412017FOFTLM2019035181004311.CSV', -'imir_190204_DoY2312017FOFTLM2019035184159965.CSV', -'imir_190204_DoY2422017FOFTLM2019035181027504.CSV', +'imir_190204_DoY2402017FOFTLM2019035180907145.CSV', +'imir_190204_DoY2412017FOFTLM2019035181004311.CSV', +'imir_190204_DoY2312017FOFTLM2019035184159965.CSV', +'imir_190204_DoY2422017FOFTLM2019035181027504.CSV', 'imir_190204_DoY2322017FOFTLM2019035184236985.CSV', -""" -filenames1 = [ -'imir_190204_DoY2432017FOFTLM2019035181100606.CSV', -'imir_190204_DoY2332017FOFTLM2019035184306076.CSV', -'imir_190204_DoY2442017FOFTLM2019035181126853.CSV', -'imir_190204_DoY2342017FOFTLM2019035184347174.CSV', -'imir_190204_DoY2452017FOFTLM2019035181155234.CSV', -'imir_190204_DoY2352017FOFTLM2019035184708935.CSV', -'imir_190204_DoY2462017FOFTLM2019035181230871.CSV', -'imir_190204_DoY2362017FOFTLM2019035184737246.CSV', -'imir_190204_DoY2472017FOFTLM2019035181252890.CSV', -'imir_190204_DoY2372017FOFTLM2019035184806338.CSV', -'imir_190204_DoY2482017FOFTLM2019035181322838.CSV', -'imir_190204_DoY2382017FOFTLM2019035184832737.CSV', -'imir_190204_DoY2492017FOFTLM2019035181406861.CSV', -'imir_190204_DoY2392017FOFTLM2019035180833486.CSV', +'imir_190204_DoY2432017FOFTLM2019035181100606.CSV', +'imir_190204_DoY2332017FOFTLM2019035184306076.CSV', +'imir_190204_DoY2442017FOFTLM2019035181126853.CSV', +'imir_190204_DoY2342017FOFTLM2019035184347174.CSV', +'imir_190204_DoY2452017FOFTLM2019035181155234.CSV', +'imir_190204_DoY2352017FOFTLM2019035184708935.CSV', +'imir_190204_DoY2462017FOFTLM2019035181230871.CSV', +'imir_190204_DoY2362017FOFTLM2019035184737246.CSV', +'imir_190204_DoY2472017FOFTLM2019035181252890.CSV', +'imir_190204_DoY2372017FOFTLM2019035184806338.CSV', +'imir_190204_DoY2482017FOFTLM2019035181322838.CSV', +'imir_190204_DoY2382017FOFTLM2019035184832737.CSV', +'imir_190204_DoY2492017FOFTLM2019035181406861.CSV', +'imir_190204_DoY2392017FOFTLM2019035180833486.CSV', 'imir_190204_DoY2502017FOFTLM2019035181519288.CSV'] -filenames = ['imir_190204_DoY2302017FOFTLM2019035184123736.CSV'] +filenames = [ +'imir_190204_DoY2462017FOFTLM2019035181230871.CSV', +'imir_190204_DoY2362017FOFTLM2019035184737246.CSV', +'imir_190204_DoY2472017FOFTLM2019035181252890.CSV', +'imir_190204_DoY2372017FOFTLM2019035184806338.CSV', +'imir_190204_DoY2482017FOFTLM2019035181322838.CSV', +'imir_190204_DoY2382017FOFTLM2019035184832737.CSV', +'imir_190204_DoY2492017FOFTLM2019035181406861.CSV', +'imir_190204_DoY2392017FOFTLM2019035180833486.CSV', +'imir_190204_DoY2502017FOFTLM2019035181519288.CSV'] -def process_file(conn, path): +def process_file(conn, path): m_raw_data = apt.mnemonics(path) + cond3, FW_volt, GW14_volt, GW23_volt, CCC_volt = whole_day_routine(m_raw_data) - FW, GW14, GW23, CCC= wheelpos_routine(m_raw_data) + FW, GW14, GW23, CCC= wheelpos_routine(m_raw_data) #put data from con3 to database - for key, value in cond3.items(): + for key, value in cond3.items(): m = m_raw_data.mnemonic(key) if value != None: if len(value) > 2: - if key == "SE_ZIMIRICEA": + if key == "SE_ZIMIRICEA": length = len(value) mean = statistics.mean(value) deviation = statistics.stdev(value) dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) sql.add_data(conn, "SE_ZIMIRICEA_HV_ON", dataset) - elif key == "IMIR_HK_ICE_SEC_VOLT4": + elif key == "IMIR_HK_ICE_SEC_VOLT4": length = len(value) mean = statistics.mean(value) deviation = statistics.stdev(value) @@ -74,34 +82,39 @@ def process_file(conn, path): mean = statistics.mean(value) deviation = statistics.stdev(value) dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) - sql.add_data(conn, key, dataset) - ######################################################################################### - if FW_volt != None: - if len(FW_volt) > 2: + sql.add_data(conn, key, dataset) +########################################################################### + + m = m_raw_data.mnemonic('IMIR_HK_FW_POS_VOLT') + if FW_volt != None: + if len(FW_volt) > 2: length = len(FW_volt) mean = statistics.mean(FW_volt) deviation = statistics.stdev(FW_volt) dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) sql.add_data(conn, "IMIR_HK_FW_POS_VOLT", dataset) - - if GW14_volt != None: - if len(GW14_volt) > 2: + + m = m_raw_data.mnemonic('IMIR_HK_GW14_POS_VOLT') + if GW14_volt != None: + if len(GW14_volt) > 2: length = len(GW14_volt) mean = statistics.mean(GW14_volt) deviation = statistics.stdev(GW14_volt) dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) sql.add_data(conn, "IMIR_HK_GW14_POS_VOLT", dataset) - - if GW23_volt != None: - if len(GW23_volt) > 2: + + m = m_raw_data.mnemonic('IMIR_HK_GW23_POS_VOLT') + if GW23_volt != None: + if len(GW23_volt) > 2: length = len(GW23_volt) mean = statistics.mean(GW23_volt) deviation = statistics.stdev(GW23_volt) dataset = (float(m.meta['start']), float(m.meta['end']), length, mean, deviation) sql.add_data(conn, "IMIR_HK_GW23_POS_VOLT", dataset) - if CCC_volt != None: - if len(CCC_volt) > 2: + m = m_raw_data.mnemonic('IMIR_HK_CCC_POS_VOLT') + if CCC_volt != None: + if len(CCC_volt) > 2: length = len(CCC_volt) mean = statistics.mean(CCC_volt) deviation = statistics.stdev(CCC_volt) @@ -110,15 +123,15 @@ def process_file(conn, path): ######################################################################################### for pos in mn.fw_positions: - try: + try: data = FW[pos] for element in data: sql.add_wheel_data(conn, 'IMIR_HK_FW_POS_RATIO_{}'.format(pos), element) except KeyError: - pass + pass for pos in mn.gw_positions: - try: + try: data_GW14 = GW14[pos] data_GW23 = GW14[pos] @@ -127,18 +140,18 @@ def process_file(conn, path): for element in data_GW23: sql.add_wheel_data(conn, 'IMIR_HK_GW23_POS_RATIO_{}'.format(pos), element) except KeyError: - pass - + pass + for pos in mn.ccc_positions: - try: + try: data = CCC[pos] for element in data: sql.add_wheel_data(conn, 'IMIR_HK_CCC_POS_RATIO_{}'.format(pos), element) except KeyError: - pass + pass + - def main(): db_file = "miri_database.db" conn = sql.create_connection(db_file) @@ -150,5 +163,5 @@ def main(): sql.close_connection(conn) print("done") -if __name__ == "__main__": - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/filterpos_extract.py b/jwql/instrument_monitors/miri_monitors/data_trending/filterpos_extract.py deleted file mode 100755 index d2558ad65..000000000 --- a/jwql/instrument_monitors/miri_monitors/data_trending/filterpos_extract.py +++ /dev/null @@ -1,112 +0,0 @@ - - -import csv_to_AstropyTable as apt -import statistics -import sqlite3 - -import mnemonics as mn -import sql_interface as sql -import condition as cond - -from collections import defaultdict - - - -#create filename string -directory='/home/daniel/STScI/trainigData/set_1_day/' - -filename= 'imir_190204_DoY2292017FOFTLM2019035182609402.CSV' - -filterpositions = [ -'FND', -'OPAQUE', -'F1000W', -'F1130W', -'F1280W', -'P750L', -'F1500W', -'F1800W', -'F2100W', -'F560W', -'FLENS', -'F2300C', -'F770W', -'F1550C', -'F2550W', -'F1140C', -'F2550WR', -'F1065C'] - -def extract_filterpos(cond, ratio_mnem, pos_mnem): - - pos_values = defaultdict(list) - - #do for every position in file - for pos in pos_mnem: - - interval = cond.get_interval(pos['time']) - - if interval is not None: - cur_pos_time = pos['time'] - filtername = pos['value'] - - for ratio in ratio_mnem: - - if (ratio['time'] > cur_pos_time) and (abs(float(ratio['value'])) > 0.1): - if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): - pos_values[pos['value']].append( ( ratio['time'], ratio['value']) ) - break - return pos_values - - - -def day_routine(path): - - #read data in file "path" and return dictionary with mnemonic and astropy table - m = apt.mnemonics(path) - - - #prepare condition for data retrieval - con_set_3 = [ - cond.greater(m.mnemonic('IMIR_HK_FW_POS_VOLT'), 250.0)] - #setup condition - condition_3=cond.condition(con_set_3) - - v = extract_filterpos(condition_3, m.mnemonic('IMIR_HK_FW_POS_RATIO'), m.mnemonic('IMIR_HK_FW_CUR_POS')) - print (v) - - - -# create initial database or check if all tables are setup - -path = directory+filename -day_routine(path) - - -print('programm end') -##################################### -#programm end - -""" - for element in m.mnemonic('IMIR_HK_FW_CUR_POS'): - - if condition_3.state(element['Secondary Time']): - cur_pos_time = element['Secondary Time'] - interval = condition_3.get_interval(element['Secondary Time'],0) - filtername = element['EU Value'] - - if interval is not None: - for item in m.mnemonic('IMIR_HK_FW_POS_RATIO'): - if (item['Secondary Time'] > cur_pos_time) and (abs(float(item['EU Value'])) > 0.1): - if (item['Secondary Time'] > interval[0]) and (item['Secondary Time'] < interval[1]): - - pos_values[] - print(filtername+'={}'.format(item['EU Value'])) - break - else: - pass - else: - pass - else: - pass -""" \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/mnemonics.py b/jwql/instrument_monitors/miri_monitors/data_trending/mnemonics.py index 18417cac9..c0bd3925d 100755 --- a/jwql/instrument_monitors/miri_monitors/data_trending/mnemonics.py +++ b/jwql/instrument_monitors/miri_monitors/data_trending/mnemonics.py @@ -1,7 +1,7 @@ -"""Module lists all neccessary mnemonics for MIRI data trending +"""Module lists all neccessary mnemonics for MIRI data trending -The module includes several lists to import to MIRI data trending monitor program. -The lists are used for data aquisation and to set up the initial database. +The module includes several lists to import to MIRI data trending monitor program. +The lists are used for data aquisation and to set up the initial database. Authors ------- @@ -9,7 +9,7 @@ Use --- -prefered import: +prefered import: import 00_mnemoncis as mn Dependencies @@ -18,10 +18,10 @@ References ---------- - + Notes ----- -The lists are for developement purpose only. +The lists are for developement purpose only. """ @@ -144,7 +144,7 @@ #grating weel positions gw_positions = [ "SHORT", -"MEDUIM", +"MEDIUM", "LONG"] #contamination control clap positions @@ -180,20 +180,20 @@ "LONG" : 441.4} gw14_nominals = { -"SHORT" : 627.49, +"SHORT" : 627.49, "MEDIUM" : 342.71, "LONG" : 408.75 } ccc_nominals = { -"LOCKED" : 577.23, +"LOCKED" : 577.23, "OPEN" : 507.86, -"CLOSED" : 399.90 } +"CLOSED" : 399.90} -#mnemonic set for basic query -#"SE_ZBUSVLT", +#mnemonic set for basic query mnemonic_set_base = [ "SE_ZIMIRICEA", +"SE_ZBUSVLT", "IMIR_HK_ICE_SEC_VOLT1", "IMIR_HK_ICE_SEC_VOLT2", @@ -283,7 +283,7 @@ "IGDP_IT_MIR_LW_STATUS", "IGDP_IT_MIR_SW_STATUS", -"IMIR_HK_FW_POS_VOLT", +"IMIR_HK_FW_POS_VOLT", "IMIR_HK_FW_POS_RATIO", "IMIR_HK_FW_CUR_POS", @@ -291,7 +291,7 @@ "IMIR_HK_GW14_POS_RATIO", "IMIR_HK_GW14_CUR_POS", -"IMIR_HK_GW23_POS_VOLT", +"IMIR_HK_GW23_POS_VOLT", "IMIR_HK_GW23_POS_RATIO", "IMIR_HK_GW23_CUR_POS", @@ -388,7 +388,7 @@ "IGDP_MIR_LW_DET_TEMP", "IGDP_MIR_SW_DET_TEMP", -"IMIR_HK_FW_POS_VOLT", +"IMIR_HK_FW_POS_VOLT", "IMIR_HK_GW14_POS_VOLT", "IMIR_HK_GW23_POS_VOLT", "IMIR_HK_CCC_POS_VOLT"] @@ -418,7 +418,7 @@ "IMIR_HK_GW14_POS_RATIO_MEDIUM", "IMIR_HK_GW14_POS_RATIO_LONG", - + "IMIR_HK_GW23_POS_RATIO_SHORT", "IMIR_HK_GW23_POS_RATIO_MEDIUM", "IMIR_HK_GW23_POS_RATIO_LONG", diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/plot_functions.py b/jwql/instrument_monitors/miri_monitors/data_trending/plot_functions.py new file mode 100644 index 000000000..624ad960a --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/plot_functions.py @@ -0,0 +1,18 @@ +import numpy as np + +def split_data(value): + x = [] + y = [] + for item in value: + x.append(item[0]) + y.append(item[1]) + X = np.array(x) + Y = np.array(y) + return X,Y + + +def pol_regression(x, y, rank): + z = np.polyfit(x, y, rank) + f = np.poly1d(z) + y_poly = f(x) + return y_poly diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/plots/__init__.py b/jwql/instrument_monitors/miri_monitors/data_trending/plots/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/plots/plot_functions.py b/jwql/instrument_monitors/miri_monitors/data_trending/plots/plot_functions.py new file mode 100644 index 000000000..624ad960a --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/plots/plot_functions.py @@ -0,0 +1,18 @@ +import numpy as np + +def split_data(value): + x = [] + y = [] + for item in value: + x.append(item[0]) + y.append(item[1]) + X = np.array(x) + Y = np.array(y) + return X,Y + + +def pol_regression(x, y, rank): + z = np.polyfit(x, y, rank) + f = np.poly1d(z) + y_poly = f(x) + return y_poly diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/plots/power_ice_plot.py b/jwql/instrument_monitors/miri_monitors/data_trending/plots/power_ice_plot.py new file mode 100644 index 000000000..54b38f9a2 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/plots/power_ice_plot.py @@ -0,0 +1,72 @@ +import sql_interface as sql +from bokeh.plotting import figure, output_file, show +from bokeh.models import BoxAnnotation +import plot_functions + +import numpy as np + +from sklearn.linear_model import LinearRegression +from sklearn.isotonic import IsotonicRegression +from sklearn.utils import check_random_state +from sklearn import metrics + +from astropy.time import Time + + + +def plot_power_ice(conn, filename): + + #query data from database + columns = ('start_time, average, deviation') + curr_idle = sql.query_data(conn, 'SE_ZIMIRICEA_IDLE', columns) + curr_hv = sql.query_data(conn, 'SE_ZIMIRICEA_HV_ON', columns) + + voltage = 30 + + #append data from query to numpy arrays + curr_idle_time, curr_idle_val = split_data(curr_idle) + power_idle = curr_idle_val * voltage + + curr_hv_time, curr_hv_val = split_data(curr_hv) + power_hv = curr_hv_val * voltage + + power_hv_reg = pol_regression(curr_hv_time, power_hv, 3) + power_idle_reg = pol_regression(curr_idle_time, power_idle, 3) + + # output to static HTML file + output_file("lines.html") + + # create a new plot with a title and axis labels + p = figure( tools = "pan,box_zoom,reset,save", \ + title = "POWER ICE", \ + y_range = [0,16], \ + x_axis_label = 'DOY (mjd)', y_axis_label='Power (W)') + + p.background_fill_color = "#efefef" + p.xgrid.grid_line_color = None + + # add a line renderer with legend and line thickness + p.scatter(curr_idle_time, power_idle, color = 'red', legend = "Power idle") + p.scatter(curr_hv_time, power_hv, color = 'orange', legend = "Power HV on ") + + p.line(curr_hv_time, power_hv_reg , color = 'green') + p.line(curr_idle_time, power_idle_reg, color = 'blue') + + p.legend.location = "bottom_right" + p.legend.click_policy = "hide" + + show(p) + +def main(): + + db_file = "miri_database.db" + conn = sql.create_connection(db_file) + + plot_power_ice(conn, "power_ice.html") + + sql.close_connection(conn) + print('end') + +#run main programm +if __name__ == "__main__": + main() diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/plots/volt4_plot.py b/jwql/instrument_monitors/miri_monitors/data_trending/plots/volt4_plot.py new file mode 100644 index 000000000..dd5230b75 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/plots/volt4_plot.py @@ -0,0 +1,63 @@ + +import data_trending.sql_interface as sql +from bokeh.plotting import figure, output_file, show +from bokeh.models import BoxAnnotation + +import plot_functions +import numpy as np + +from astropy.time import Time + + +def plot_power_ice(conn, filename): + + #query data from database + columns = ('start_time, average, deviation') + volt4_idle = sql.query_data(conn, 'IMIR_HK_ICE_SEC_VOLT4_IDLE', columns) + volt4_hv = sql.query_data(conn, 'IMIR_HK_ICE_SEC_VOLT4_HV_ON', columns) + voltage = 30 + #append data from query to numpy arrays + + volt4_idle_time, volt4_idle_val = split_data(volt4_idle) + volt4_hv_time, volt4_hv_val = split_data(volt4_hv) + + volt4_hv_reg = pol_regression(volt4_hv_time, volt4_hv_val, 3) + volt4_idle_reg = pol_regression(volt4_idle_time, volt4_idle_val, 3) + + # output to static HTML file + output_file("lines.html") + + # create a new plot with a title and axis labels + p = figure( tools = "pan,box_zoom,reset,save", \ + title = "Volt 4", \ + y_range = [4,5], + x_axis_label = 'DOY (mjd)', y_axis_label= 'Voltage (V)') + + p.background_fill_color = "#efefef" + p.xgrid.grid_line_color = None + + # add a line renderer with legend and line thickness + p.scatter(volt4_idle_time, volt4_idle_val, color = 'red', legend = "Power idle") + p.scatter(volt4_hv_time, volt4_hv_val, color = 'orange', legend = "Power HV on ") + + p.line(volt4_hv_time, volt4_hv_reg , color = 'green') + p.line(volt4_idle_time, volt4_idle_reg, color = 'blue') + + p.legend.location = "bottom_right" + p.legend.click_policy = "hide" + + show(p) + +def main(): + + db_file = "miri_database.db" + conn = sql.create_connection(db_file) + + plot_power_ice(conn, "volt4.html") + + sql.close_connection(conn) + print('end') + +#run main programm +if __name__ == "__main__": + main() diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/plotting.py b/jwql/instrument_monitors/miri_monitors/data_trending/plotting.py deleted file mode 100755 index ff418f846..000000000 --- a/jwql/instrument_monitors/miri_monitors/data_trending/plotting.py +++ /dev/null @@ -1,92 +0,0 @@ -import sql_interface as sql -import mnemonics as mn - -from os.path import join, dirname -import datetime - -import pandas as pd -from scipy.signal import savgol_filter - -from bokeh.io import curdoc -from bokeh.layouts import row, column -from bokeh.models import ColumnDataSource, DataRange1d, Select -from bokeh.palettes import Blues4 -from bokeh.plotting import figure - -STATISTICS = ['record_min_temp', 'actual_min_temp', 'average_min_temp', 'average_max_temp', 'actual_max_temp', 'record_max_temp'] - -def get_dataset(src, name, distribution): - df = src[src.airport == name].copy() - del df['airport'] - df['date'] = pd.to_datetime(df.date) - # timedelta here instead of pd.DateOffset to avoid pandas bug < 0.18 (Pandas issue #11925) - df['left'] = df.date - datetime.timedelta(days=0.5) - df['right'] = df.date + datetime.timedelta(days=0.5) - df = df.set_index(['date']) - df.sort_index(inplace=True) - if distribution == 'Smoothed': - window, order = 51, 3 - for key in STATISTICS: - df[key] = savgol_filter(df[key], window, order) - - return ColumnDataSource(data=df) - -def make_plot(source, title): - plot = figure(x_axis_type="datetime", plot_width=800, tools="", toolbar_location=None) - plot.title.text = title - - plot.quad(top='record_max_temp', bottom='record_min_temp', left='left', right='right', - color=Blues4[2], source=source, legend="Record") - plot.quad(top='average_max_temp', bottom='average_min_temp', left='left', right='right', - color=Blues4[1], source=source, legend="Average") - plot.quad(top='actual_max_temp', bottom='actual_min_temp', left='left', right='right', - color=Blues4[0], alpha=0.5, line_color="black", source=source, legend="Actual") - - # fixed attributes - plot.xaxis.axis_label = None - plot.yaxis.axis_label = "Temperature (F)" - plot.axis.axis_label_text_font_style = "bold" - plot.x_range = DataRange1d(range_padding=0.0) - plot.grid.grid_line_alpha = 0.3 - - return plot - -def update_plot(attrname, old, new): - city = city_select.value - plot.title.text = "Weather data for " + cities[city]['title'] - - src = get_dataset(df, cities[city]['airport'], distribution_select.value) - source.data.update(src.data) - -city = 'Austin' -distribution = 'Discrete' - -cities = { - 'Austin': { - 'airport': 'AUS', - 'title': 'Austin, TX', - }, - 'Boston': { - 'airport': 'BOS', - 'title': 'Boston, MA', - }, - 'Seattle': { - 'airport': 'SEA', - 'title': 'Seattle, WA', - } -} - -city_select = Select(value=city, title='City', options=sorted(cities.keys())) -distribution_select = Select(value=distribution, title='Distribution', options=['Discrete', 'Smoothed']) - -df = pd.read_csv(join(dirname(__file__), 'data/2015_weather.csv')) -source = get_dataset(df, cities[city]['airport'], distribution) -plot = make_plot(source, "Weather data for " + cities[city]['title']) - -city_select.on_change('value', update_plot) -distribution_select.on_change('value', update_plot) - -controls = column(city_select, distribution_select) - -curdoc().add_root(row(plot, controls)) -curdoc().title = "Weather" \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/test_query_data.py b/jwql/instrument_monitors/miri_monitors/data_trending/test_query_data.py index 7c8a41dff..b32d53ca2 100755 --- a/jwql/instrument_monitors/miri_monitors/data_trending/test_query_data.py +++ b/jwql/instrument_monitors/miri_monitors/data_trending/test_query_data.py @@ -1,6 +1,6 @@ -import sql_interface as sql +import sql_interface as sql from bokeh.plotting import figure, output_file, show -from bokeh.models import BoxAnnotation +from bokeh.models import BoxAnnotation import numpy as np @@ -17,7 +17,7 @@ conn = sql.create_connection(db_file) -query_name = 'IGDP_MIR_LW_V_VDDUC' +query_name = 'IMIR_IC_SCE_DIG_TEMP' columns = ('start_time, average, deviation') queried_data = sql.query_data(conn,query_name, columns) @@ -25,7 +25,7 @@ x=[] y=[] -for item in queried_data: +for item in queried_data: x.append(item[0]) y.append(item[1]) #################################################### @@ -56,24 +56,17 @@ # add a line renderer with legend and line thickness p.scatter(x, y, color='red', legend = "Data points") p.line(x, y_, color='green', legend= "Isotonic regression") -p.line(X, y_pred, color='blue', legend= "Lin. regression") +#p.line(X, y_pred, color='blue', legend= "Lin. regression") -print('Mean Absolute Error:', metrics.mean_absolute_error(Y, y_pred)) -print('Mean Squared Error:', metrics.mean_squared_error(Y, y_pred)) -print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(Y, y_pred))) +print('Mean Absolute Error:', metrics.mean_absolute_error(Y, y_pred)) +print('Mean Squared Error:', metrics.mean_squared_error(Y, y_pred)) +print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(Y, y_pred))) #p.line(x, y1, legend="deviation", line_color="orange", line_dash="4 4") -p.legend.location = "bottom_right" -p.legend.click_policy="hide" +p.legend.location = "top_right" +p.legend.click_policy = "hide" # show the results show(p) print('end') - - - - - - - diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/test_query_pos.py b/jwql/instrument_monitors/miri_monitors/data_trending/test_query_pos.py deleted file mode 100755 index 42d643681..000000000 --- a/jwql/instrument_monitors/miri_monitors/data_trending/test_query_pos.py +++ /dev/null @@ -1,79 +0,0 @@ -import sql_interface as sql -from bokeh.plotting import figure, output_file, show -from bokeh.models import BoxAnnotation - -import numpy as np - -from sklearn.linear_model import LinearRegression -from sklearn.isotonic import IsotonicRegression -from sklearn.utils import check_random_state -from sklearn import metrics - -from astropy.time import Time - -############################################### - -db_file = "miri_database.db" -conn = sql.create_connection(db_file) - - -query_name = 'IMIR_HK_FW_POS_RATIO_F1065C' - -columns = ('timestamp, value') -queried_data = sql.query_pos(conn, query_name, columns) - -x=[] -y=[] - -for item in queried_data: - x.append(item[0]) - y.append(item[1]) -#################################################### - -X=np.array(x) -Y=np.array(y) - -ir = IsotonicRegression() -y_ = ir.fit_transform(X,Y) - -lr = LinearRegression() -lr.fit(X[:, np.newaxis], Y) # x needs to be 2d for LinearRegression -y_pred = lr.predict(X[:, np.newaxis]) -# output to static HTML file -output_file("lines.html") - -# create a new plot with a title and axis labels -p = figure( tools = "pan,box_zoom,reset,save", \ - title = query_name, \ - x_axis_label = 'DOY', y_axis_label='data') - -p.background_fill_color = "#efefef" -p.xgrid.grid_line_color = None -#p.add_layout(BoxAnnotation(top=0.23, fill_alpha=0.1, fill_color='red', line_color='red')) -#p.add_layout(BoxAnnotation(bottom=0.24, fill_alpha=0.1, fill_color='red', line_color='red')) - - -# add a line renderer with legend and line thickness -p.scatter(x, y, color='red', legend = "Data points") -p.line(x, y_, color='green', legend= "Isotonic regression") -p.line(X, y_pred, color='blue', legend= "Lin. regression") - -print('Mean Absolute Error:', metrics.mean_absolute_error(Y, y_pred)) -print('Mean Squared Error:', metrics.mean_squared_error(Y, y_pred)) -print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(Y, y_pred))) -#p.line(x, y1, legend="deviation", line_color="orange", line_dash="4 4") - -p.legend.location = "bottom_right" -p.legend.click_policy="hide" - -# show the results -show(p) - -print('end') - - - - - - - diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/trash.py b/jwql/instrument_monitors/miri_monitors/data_trending/trash.py deleted file mode 100755 index bd1f5598c..000000000 --- a/jwql/instrument_monitors/miri_monitors/data_trending/trash.py +++ /dev/null @@ -1,275 +0,0 @@ - - - -def update_task(conn, task): - """ - update priority, begin_date, and end date of a task - :param conn: - :param task: - :return: project id - - """ - sql = ''' UPDATE tasks - SET priority = ? , - begin_date = ? , - end_date = ? - WHERE id = ?''' - - cur = conn.cursor() - cur.execute(sql, task) - -def create_project(conn, project): - """ - Create a new project into the projects table - :param conn: - :param project: - :return: project id - """ - sql = ''' INSERT INTO projects(name,begin_date,end_date) - VALUES(?,?,?) ''' - cur = conn.cursor() - cur.execute(sql, project) - return cur.lastrowid - -def create_task(conn, task): - """ - Create a new task - :param conn: - :param task: - :return: - """ - - sql = ''' INSERT INTO tasks(name,priority,status_id,project_id,begin_date,end_date) - VALUES(?,?,?,?,?,?) ''' - cur = conn.cursor() - cur.execute(sql, task) - return cur.lastrowid - -def add_data(conn, name, dataset): - - cur=conn.cursor() - cur.execute('INSERT INTO {} (start_time,end_time,datapoints,average,deviation) \ - VALUES (?,?,?,?,?)'.format(name),dataset) - conn.commit() - - - - - return cur.lastrowid - -def delete_task(conn, id): - """ - Delete a task by task id - :param conn: Connection to the SQLite database - :param id: id of the task - :return: - """ - sql = 'DELETE FROM tasks WHERE id=?' - cur = conn.cursor() - cur.execute(sql, (id,)) - - -def delete_all_tasks(conn): - """ - Delete all rows in the tasks table - :param conn: Connection to the SQLite database - :return: - """ - sql = 'DELETE FROM tasks' - cur = conn.cursor() - cur.execute(sql) - - - -def select_all_tasks(conn): - """ - Query all rows in the tasks table - :param conn: the Connection object - :return: - """ - cur = conn.cursor() - cur.execute("SELECT * FROM tasks") - - rows = cur.fetchall() - - for row in rows: - print(row) - - - -import functions as func -import statistics -import sqlite3 - -import mnemonics as mn -import sql_interface as sql -import condition as cond - -#module does: -# -# -# -# - -#create filename string -directory='/home/daniel/STScI/trainigData/set_1_15min/' - -filenames=[ -'imir_190130_otis229FOFTLM2019030204146194.CSV', -'imir_190130_otis240FOFTLM2019030210631185.CSV', -'imir_190130_otis230FOFTLM2019030204240886.CSV', -'imir_190130_otis241FOFTLM2019030210651672.CSV', -'imir_190130_otis231FOFTLM2019030204334644.CSV', -'imir_190130_otis242FOFTLM2019030210728909.CSV', -'imir_190130_otis232FOFTLM2019030204455835.CSV', -'imir_190130_otis243FOFTLM2019030210744062.CSV', -'imir_190130_otis233FOFTLM2019030204521412.CSV', -'imir_190130_otis244FOFTLM2019030210809362.CSV', -'imir_190130_otis234FOFTLM2019030204555665.CSV', -'imir_190130_otis245FOFTLM2019030210828095.CSV', -'imir_190130_otis235FOFTLM2019030204617145.CSV', -'imir_190130_otis246FOFTLM2019030210852965.CSV', -'imir_190130_otis236FOFTLM2019030204651604.CSV', -'imir_190130_otis247FOFTLM2019030210914141.CSV', -'imir_190130_otis237FOFTLM2019030204712019.CSV', -'imir_190130_otis248FOFTLM2019030210940944.CSV', -'imir_190130_otis238FOFTLM2019030204738855.CSV', -'imir_190130_otis249FOFTLM2019030211002524.CSV', -'imir_190130_otis239FOFTLM2019030204805611.CSV', -'imir_190130_otis250FOFTLM2019030211032094.CSV'] - - -def once_a_day_routine(path, conn): - - #read data in file "path" and return dictionary with mnemonic and astropy table - m = func.mnemonics(path) - - #prepare condition for data retrieval - con_set_1 = [ \ - cond.equal(m.mnemonic('IMIR_HK_IMG_CAL_LOOP'),'OFF'), \ - cond.equal(m.mnemonic('IMIR_HK_IFU_CAL_LOOP'),'OFF'), \ - cond.equal(m.mnemonic('IMIR_HK_POM_LOOP'),'OFF'), \ - cond.smaller(m.mnemonic('IMIR_HK_ICE_SEC_VOLT1'),1), \ - cond.greater(m.mnemonic('SE_ZIMIRICEA'),0.2)] - #setup condition - condition_1=cond.condition(con_set_1) - - - for mnemonic_nam in mn.mnemonic_cond_1: - - dataset=func.extract_useful_data(condition_1, m.mnemonic(mnemonic_nam)) - - if dataset is None: - print('no data for {}'.format(mnemonic_nam)) - else: - sql.add_data(conn,mnemonic_nam, dataset) - - del condition_1 - - print('Values for condition 1 extracted: {}'.format(filename)) - - # - # - # - # - # - # - # - - #prepare condition for data retrieval - con_set_2 = [ \ - cond.greater(m.mnemonic('SE_ZIMIRFPEA'),0.5), \ - cond.equal(m.mnemonic('IGDP_IT_MIR_IC_STATUS'),'DETECTOR READY'), \ - cond.equal(m.mnemonic('IGDP_IT_MIR_LW_STATUS'),'DETECTOR READY')] - #setup condition - condition_2=cond.condition(con_set_2) - - for mnemonic_nam in mn.mnemonic_cond_2: - - dataset=func.extract_useful_data(condition_2, m.mnemonic(mnemonic_nam)) - - if dataset is None: - print('no data for {}'.format(mnemonic_nam)) - else: - sql.add_data(conn,mnemonic_nam, dataset) - - del condition_1 - - print('Values for condition 2 extracted: {}'.format(filename)) - - - - -# create initial database or check if all tables are setup -db_file = "MIRI_mnemonics.db" -conn = sql.create_connection(db_file) -sql.create_database(conn, mn.mnemonic_set) - -path = directory + filenames[0] -once_a_day_routine(path,conn) - -sql.close_connection(conn) -print('programm end') -##################################### -#programm end -def extract_useful_data(condition, mnemonic): - - temp = [] - - #look for all values that fit to the given conditions - for element in mnemonic: - # print('at %6.8f appears %6.8f',element['Primary Time'],element['EU Value'])) - if condition.state( float(element['Secondary Time'])): - - print("condition true: value= {}, time= {}".format(str(element['EU Value']),str(element['Secondary Time']))) - - temp.append(float(element['EU Value'])) - - length= len(temp) - - - if length > 2: - average = sum(temp) / length - deviation=statistics.stdev(temp) - del temp[:] - return (float(mnemonic['Secondary Time'][0]), float(mnemonic['Secondary Time'][-1]), length, average, deviation) - else: - return None - - - - -def extract_filterpos(condition, ratio_mnem, pos_mnem): - '''Extracts to filterpositions corresponding ratio values - Parameters - ---------- - condition : object - conditon object that holds one or more subconditions - ratio_mem : AstropyTable - holds ratio values of one specific mnemonic - pos_mem : AstropyTable - holds pos values of one specific mnemonic - Return - ------ - pos_values : dict - holds ratio values and times with corresponding pos label - ''' - - pos_values = defaultdict(list) - - #do for every position in mnemonic - for pos in pos_mnem: - - interval = condition.get_interval(pos['time']) - - if interval is not None: - cur_pos_time = pos['time'] - filtername = pos['value'] - - for ratio in ratio_mnem: - - if (ratio['time'] == cur_pos_time) and (abs(float(ratio['value'])) > 0.1): - if (ratio['time'] > interval[0]) and (ratio['time'] < interval[1]): - pos_values[pos['value']].append( ( ratio['time'], ratio['value']) ) - break - return pos_values diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/volt1-3.html b/jwql/instrument_monitors/miri_monitors/data_trending/volt1-3.html new file mode 100644 index 000000000..15987a17c --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/volt1-3.html @@ -0,0 +1,60 @@ + + + + + + Bokeh Plot + + + + + + + + +

+
+
+ + + + + \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/volt1-3.py b/jwql/instrument_monitors/miri_monitors/data_trending/volt1-3.py new file mode 100644 index 000000000..f24e4e8e0 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/data_trending/volt1-3.py @@ -0,0 +1,64 @@ +import sql_interface as sql +from bokeh.plotting import figure, output_file, show +from bokeh.models import BoxAnnotation +import plot_functions as pf + +import numpy as np + +from astropy.time import Time + +def plot_volt1_3(conn, filename): + + #query data from database + columns = ('start_time, average, deviation') + volt1 = sql.query_data(conn, 'IMIR_HK_ICE_SEC_VOLT1', columns) + volt2 = sql.query_data(conn, 'IMIR_HK_ICE_SEC_VOLT2', columns) + volt3 = sql.query_data(conn, 'IMIR_HK_ICE_SEC_VOLT3', columns) + + #append data from query to numpy arrays + volt1_time, volt1_val = pf.split_data(volt1) + volt2_time, volt2_val = pf.split_data(volt2) + volt3_time, volt3_val = pf.split_data(volt3) + + volt1_reg = pf.pol_regression(volt1_time, volt1_val, 3) + volt2_reg = pf.pol_regression(volt2_time, volt2_val, 3) + volt3_reg = pf.pol_regression(volt3_time, volt3_val, 3) + + # output to static HTML file + output_file("volt1-3.html") + + # create a new plot with a title and axis labels + p = figure( tools = "pan,box_zoom,reset,save", \ + title = "IMIR_HK_ICE_SEC_VOLT 1 - 3", \ + x_axis_label = 'DOY (mjd)', y_axis_label='Power (W)') + + p.background_fill_color = "#efefef" + p.xgrid.grid_line_color = None + + # add a line renderer with legend and line thickness + p.scatter(volt1_time, volt1_val, color = 'red', legend = "Volt 1") + p.scatter(volt2_time, volt2_val, color = 'orange', legend = "Volt 2") + p.scatter(volt3_time, volt3_val, color = 'purple', legend = "Volt 3") + + p.line(volt1_time, volt1_reg, color = 'green') + p.line(volt2_time, volt2_reg, color = 'green') + p.line(volt3_time, volt3_reg, color = 'green') + + p.legend.location = "bottom_right" + p.legend.click_policy = "hide" + + show(p) + +def main(): + + db_file = "miri_database.db" + conn = sql.create_connection(db_file) + + plot_volt1_3(conn, "power_ice.html") + + sql.close_connection(conn) + print('end') + +#run main programm +if __name__ == "__main__": + main() diff --git a/jwql/tests/test_edb_interface.py b/jwql/tests/test_edb_interface.py deleted file mode 100644 index 4f17fc197..000000000 --- a/jwql/tests/test_edb_interface.py +++ /dev/null @@ -1,46 +0,0 @@ -#! /usr/bin/env python -"""Tests for the ``engineering_database`` module. - -Authors -------- - - - Johannes Sahlmann - - -Use ---- - - These tests can be run via the command line (omit the ``-s`` to - suppress verbose output to ``stdout``): - - :: - - pytest -s test_edb_interface.py -""" - -from astropy.time import Time -import pytest - - -@pytest.mark.xfail -def test_get_all_mnemonics(): - """Test the retrieval of all mnemonics.""" - - from ..utils.engineering_database import get_all_mnemonic_identifiers - - all_mnemonics = get_all_mnemonic_identifiers()[0] - assert len(all_mnemonics) > 1000 - - -@pytest.mark.xfail -def test_query_single_mnemonic(): - """Test the query of a mnemonic over a given time range.""" - - from ..utils.engineering_database import query_single_mnemonic - - mnemonic_identifier = 'SA_ZFGOUTFOV' - start_time = Time(2016.0, format='decimalyear') - end_time = Time(2018.1, format='decimalyear') - - mnemonic = query_single_mnemonic(mnemonic_identifier, start_time, end_time) - assert len(mnemonic.data) == mnemonic.meta['paging']['rows'] diff --git a/jwql/website/db.sqlite3 b/jwql/website/db.sqlite3 index 0902712bb80a8f318932a12fb336e4adc8f18889..8a9326ec84930eb9bc18b8ca57d8450da0aa0904 100644 GIT binary patch delta 2426 zcmah~eQX;^72ol$6R+*v=jNj!AB~-mCe5Wuc5Sa6=hRExd^%Srt=lGcIOJ|@$6jas zQQP?xkc3u;bX1~^C6+r5?gW*nkbnbA%^!CsdRI6>DFu#`ih9=ufq)`Ws`dn@Lr7uP zX&nMWtTazEzxmCZc{6W+Z}qBf^{Vbd{T@F-5Y4zX;)Za0?&3k4)=zXDJof$YS3y_L zV5~+@<;zW-8a-8Tn{Lw2dTO*p&7@llC!{3DakikGibj&T%u-Mchl64$C#90X8Ci@5 zL$Q<;ie#-E@9N~Not(qM3%xd1ui)z9c_-%*3gdMJZPgEnG6-s64!#7J;XU|01R+>x z;+i4f)7_@2s>;9BT~$+!$Ziyli^*szn2@3wQQan6lRw{Gzd~1P$qppWY;KE zO%c`RU5oM~<8O>KGpz(XfZOn6<_mBIF2Oo;3_c4H@G@uM5IhAn%xOUEJ@yxji+z*5 z%;uR^c7=_x$5|U&!8~N{Fz+zmV_qsWS#BFwDk!ucmCZ|$ghkQr%;uzEQb6NQ=(RUAh%skyYdnnE3jECM?>-B>}Q6wCXhMH3EF>H5}gt zyRv4~FvMCrLDw?fH0@^6)3f0jYcLxVt%9S4x8fw_S{w_<ytKNhu<$c4jW>5)RAKe56av zJxOYwH(7XlFYoBJ3SCZ{-D!8^eXsmLr-6F}+=CAbdrp4?RQY4M`sa?9EXzPo% zu4rrXxy=iD(l|}9Z!_nN)7alht0Iq2#MKf)D~KvO+{jv`Ord$7tYKi zbMr#VA&P;-nB-kPl9)Q_;ilq)oOgIM9+;en2jX(fyBrY4aNm1Uj(Z2>*jPMZ4=hi} z-hs5!BfD||{}|`D4)NjP*i67b!HL6*>45CQ>latDfy7W;*-fj9jWxPkIonMn@Lx+^F z&7y;-S-C!oPAFmmH7a#7>Qm0j=nu+i8ClqN%vSXde`}^bB^XB+Hy(;;R!0$pCV|f< zX~?w$&nS=L=rYn5(CIDnH`1mnuOiAqHtQFa@~eoiGTb(N$FORcH1LLU{h#zd)xWG? z#2Iu@ztY^SCD8uPT{6@!CM~K&SRTt#M-%3BI{!T70b_&c>~=*Tcm0GMT)io+j^`g z4({4yr?f@_^Mmt+02HX}?ED6K-&-lh{{JvGP0B z$}c#2?R*#Sbn$jh>A!Q zLot?rV8_a>yXcdK0a8Ufi|FA^jG8x5lVM7uqH~OT-j`K0vWbMYa&iQzxUef^tA%&< Ta%yGh5}bnIawu5J;lqLp3{T9xO*i7(eJcpO@J3NU|Y{nP4I|cRT zEoX8?Y$7;^H}Mjl#S{1f_F)sxx>0XxJ?kxy76{Jcb)3a%d<&1`vlzk_tj99UN04vH zU&!a=DOs0?y~UEBGm1;0~wpeSDicc^xOYn=w3s z!}v7DxnLdY@tfl!l}3qtK^o;Q2BqZa+;Cf&B*X}=m#Fr+7(bc=CFUEK|p+2wo++3*WNQErijt*=<-EsvYW#fw=dOqWdS4TFY&FebFp z4>>ZcH`rf#V&tij=t#6KE5EiD4ka>+F3)@A%k&@N;>NZ>!`^^%Z^O2S0?xH7DfVs` zOjZ*DkRJ%Zp`riN&phJF^3*e35Eg|Idl9?c2@|yB0rp4~ri8~IIvd)9TC-<2( z1(#t_n6}f;ZDt%1%;jKi@UeNB*0Pc!YTw{-D}I;T<5E-u5qgsevj%^{mr78 zU{$7o9Vns?E~D-$r{YVaUZp~*s#d)2U=~Le*w3}u!WxwCG2PsU2W?(@nm}@ z>0uMdBRh8P2}Tp~idbLwU2vX*)_&tM$i|Tt{+s)U1=Uldsufqpl^i+Cht%!@*a7jPhs#r1UckIjC==ko9+O+)EO? zJ-!OJPw{(wO~7H_lEe)M-M~KP61MbDf?|qb3cuwa(mQBaFSLL@Hs@qF=Al%iI79Gz zoZ-0?re+qQ-frh5nGJX1Y57n27$4e@Yz-StBu5=_@lLUeGVPR_3`Vj_)pj?xHEI#& mMOts!N{yUpPb0HP{pmlddzIRL7kLipIw>HetM=au)%*o7agtF0 From 7eaab8863e8f76469d926b4b87c0a9e0307757ab Mon Sep 17 00:00:00 2001 From: DanielKuebi Date: Fri, 15 Feb 2019 15:56:32 -0500 Subject: [PATCH 011/174] added output folder --- .../miri_monitors/__init__.py | 0 .../miri_monitors/data_trending/day_to_db.py | 12 +-- .../data_trending/plots/miri_database.db | Bin 0 -> 466944 bytes .../data_trending/plots/output/volt4.html | 60 +++++++++++++++ .../data_trending/plots/volt4_plot.py | 20 ++--- .../data_trending/sql_interface.py | 72 ++++++++---------- 6 files changed, 104 insertions(+), 60 deletions(-) create mode 100644 jwql/instrument_monitors/miri_monitors/__init__.py create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/plots/miri_database.db create mode 100644 jwql/instrument_monitors/miri_monitors/data_trending/plots/output/volt4.html diff --git a/jwql/instrument_monitors/miri_monitors/__init__.py b/jwql/instrument_monitors/miri_monitors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py b/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py index a65c9ee6f..2363fdae3 100755 --- a/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py +++ b/jwql/instrument_monitors/miri_monitors/data_trending/day_to_db.py @@ -15,7 +15,7 @@ #create filename string directory='/home/daniel/STScI/trainigData/set_1_day/' -filenames1 = [ +filenames = [ 'imir_190204_DoY2292017FOFTLM2019035182609402.CSV', 'imir_190204_DoY2402017FOFTLM2019035180907145.CSV', 'imir_190204_DoY2412017FOFTLM2019035181004311.CSV', @@ -38,16 +38,6 @@ 'imir_190204_DoY2392017FOFTLM2019035180833486.CSV', 'imir_190204_DoY2502017FOFTLM2019035181519288.CSV'] -filenames = [ -'imir_190204_DoY2462017FOFTLM2019035181230871.CSV', -'imir_190204_DoY2362017FOFTLM2019035184737246.CSV', -'imir_190204_DoY2472017FOFTLM2019035181252890.CSV', -'imir_190204_DoY2372017FOFTLM2019035184806338.CSV', -'imir_190204_DoY2482017FOFTLM2019035181322838.CSV', -'imir_190204_DoY2382017FOFTLM2019035184832737.CSV', -'imir_190204_DoY2492017FOFTLM2019035181406861.CSV', -'imir_190204_DoY2392017FOFTLM2019035180833486.CSV', -'imir_190204_DoY2502017FOFTLM2019035181519288.CSV'] def process_file(conn, path): m_raw_data = apt.mnemonics(path) diff --git a/jwql/instrument_monitors/miri_monitors/data_trending/plots/miri_database.db b/jwql/instrument_monitors/miri_monitors/data_trending/plots/miri_database.db new file mode 100644 index 0000000000000000000000000000000000000000..d4b199c935a78921e72210314aae2045b8b09a30 GIT binary patch literal 466944 zcmeF41zc2H_xA^+6ovss2}My!i>?7&hwf%j1O)^{ED#Y}R2s3n#lr4R>=q0JvAa;Q z+j;i@DxA%1Zg8Lf^WJCk`CKlZIX~7p-&q{^?X~vaf&N}`NikY|;u8l(CTW>+lsFO+ z97aov!;yZ;;cybse?LDW(HBxb|AGE5QR8bdU!w2PMQffkdX>l9Av!04_m%g7_nKGE zd(6AXyU8o%UFKcjo#q|m9pde+Iz`O4qK7E~N`Mle1SkPYfD)htC;>`<5}*Vqf&W1Q zv7zXudkN4dfe-rRm>&xlny>fh z$vlUk4&{AQF^3y;Hz7ZFggSiw(Hk z*pByG9$+ZEtTqk>Hxjtztm72G`3o=6$1Cu7HvLq|OH)j4WM-%MTkcmVEYUSX(TfC< z8qS<;aDKH^>v*+6;6B{nCrr?dCAZ+4TiS7d%L91Ak}F0ix{{!SmnUZjiazPYGVMv= z`QZI2*;5m6xv`zXZ@J$l;d!nzioRLka$JdX5P^?bX?JuWctp9Ka*ayFfTjcZfw`&x7=@vu<)P|MV~5YBYB^5 z0M4)7o!uwN1(9F=DIs_TCO0*=Yy4a8Unnf%okY=#1m0uaIQc00+?zIQ27u=Tha&Su z%DCLfPVTqdFJ5@T@jZ&3An@sSlyetF?=b%Ugc;ygSo)*i?L16wVrD1%TkgL}SahQp zMc*tiH9O3?0_WEZj@x9%fcwm5Co(lxV{&6FJDD2tDZ+~?N05A~!0SLh=K+d7dPA4Y zq2T)9>_9oy&zRiE+D^KLJYHC2KL*JY1p2W{IJ@Bdo-3Q@sci(WIVW8_^@4G^sjZYq zUiA~>=O^9v+9P!DegfSDA3wOzYQ3DX?lYKuGeo+!AH zK&O8o=Q7HgPPy&WM;xQF|`zp6f~+d^0QC$}xg!t9eIqmQS`lr`4cNJMMiChU@BG z^)kP>!`e+YT`b2!^xOetkW+-0Iz==Ar;&GEH>!*R=h>s0jq`rVQk(M4tt~}k1J$tnmk3Mx zP3GL2YyoA1rCxR_SPSdU?N60gjKqW{rdFa+fuFa_G-~$Eg>x@eQVOok46wbR{2ErM zYRa?^V!c16CVX>aOVNP9&yeB5k{9xvd$~=ZWTT4cY9Y{9?zXV|NLEN=zKLKEvS1-hK!9}t2ru>uc;LMygGX3_0r8eT5TH00)1yoZ8VewOMw`R-%!B>f7~iBfR_&amwLrfa`u6e8)0BVDrLJ9==D; z)K;2U^G(gHM1ug;O5a1m(;n?O_fjNmp{!A_f<=p+;E04<%9HC`SxPIEQ6m43n*Qkr zB|r&K0+awHKnYL+lmI0_2~Yx*03|>P{EG>Q{J#R%QdIw+#Pj`C|JUI@uB!k4i*KFY z8YMspPy&;IOO_5bQR{LlLTFvhW5 zvs179b4*i*``0EOT*VUN<^Lwh^EG~!|9R-5B~~~&Il=b*r8nGqIkVKF@_((l0kz8i zoLt6nN2qmp^VhIdcXW#Iek+z7)&H&Ptp7JfaaDY*?oWp8IkLwpnxtYvcKu%_YuqtZ z|ChMNI4;t9yt3(QSbwwQ?5{m!SxT|>|3-}C;=Dsbp%51P`xh6g9A*ho{oly0aw8_UsF>^?yzpTi~7YL|!|95JR9kgZw zZ%AxWR#t{x7q>q5w92Zjt{OWK%>bwl`)lq%FD9_4uW!V37ZCPsa3;-MS9=&MSG3p#_ zFL*LQDqjOrix~j0V*;;#3E4h$0_-wM*Nz=nURzi>0l>0y0zeg~UlRb5nZR3}XWJfp z1|?}~+ZMh%#uEPY|JC#VMfHDW`<5}*Vq0ZQP% zj)3_2|4kUjid6;Zf+1(D94{}WU2WTh&;S4F;kz)7udaNSZyOC8zF%Fk?+DA`)l>i1 zN9xmy=FOWAn--r{pJ&;*c3AQA|2bRH1#4_lzi$hfOVw{q^S5Rx>umt2_}E3XEaTQR zB}KQZ&sgeu8UV5xr$$#^e{*z(4a+xVDyXw9wb%s!)X@E^Y!C_eaz6z^cN@n$VIfA7mH2_p-?GS~WB;D$}|IUDKel26e;tp3nX ztDNYpbX^%)RZB~W?EXoSy$8g&d%FjOyLpDYJ35C4Iy;63`+5bL)_G9YF~HeA$XP4M z-oeXRtIkuj^tJx%M_ja)yHAj_t8+l7KP#GAS}VAwF(U(vZJ(L6ONZiIK5?^QI7~=$PcV$fUUVL4WINw6qdp5=C22#zcps z-6wdWe;=oLNb|7a@=#M!R@K;8B0J_+9*Uli=uc4l zaJS%aUmsE4)jrY)%gfrY)BL3Q*>L%(At9@3Xef~#h3BWclb3U?^b{SC6rY`-<|uk8Dsqds)3|}$<_aYW^AtiAH05v0=l!8k0eTW8 zKnYL+lmI0_2~Yx*03|>P{B;7hm74)pcR<(L3^0Xh+IDwOe$q3D_`0pdtEYQwR}|~n z3vejYv|ShDldTe=xA(W+BO_cfwXwBroym{u3SyLf0*4` zY{@q^wyoR>uu5HfE5Lr}%D+lo$pnZ@{GPd@>;e`WEjeGg6JT}hS~~%nGEFs3x}Xo} zzE)y!&(Mb~wP+*2TGjnp8v(|m`}S0vuXY`JTg5N9VYeJto7h(F16Uoq);@p((3N)# z5OutV#p_(VmDmwen_1MH;$M3kz$iqm%jh<;M<7ybxa7KQ2V70s1uz{2-=x&~oE${V zG;7Ccd<&DCn%7$Azt%2*NhtVWh0T+?LU`QiqKzlYaJ5mbb^dE@0vLhRJvBu)9V5E*x zTdtV|5#1M#*rdps-GJ1@9RRb?J03GWF0LO$&f1&i^Pm}a=|-#_02}w$?*DTKz!0Ri z^@%*khu*=?D@@7-EH&Bz(7M*F|5`f$jzpK<neF;#PYDKppghax6lm?q2ZfT2xT>o8%PYvL>gY&hhg1z{1%>wrax@VQxjF>a_?lOH7^}uBbOeizU^UnX zR(hw#vZ^*V64~*U+34=-{lpwrsFkB)PNL~b)3xE!)k;oQ z)zMKRdstPv*3mF!R#mpDtH@vdl6>v775_bG>eAG0xYRY5l~uL3m&hJUNL^8ds_x7c zz0&v(BrdIS{mNO<7qp}5Z+BG5%j18q8jJp3wXgbnwd!ldTDe*C75WtYm60?X|9&<$ z5tEJ8?x`{x9k}AM(Sc?o&Bg}%oi3{X*O>pWq*$W3NHLnu{}(#~LSLW+C;>`<5}*Vq z0ZM=ppadv^|NjWo>JSJQ77BVYRPAalVgCOOq~y0Xc+d<&0`xk`?`JJoU`n3< zFZb(qd$ijBIj7j16}NH`zZy#b`qf&*&xdK2^}$bc&mq||a)*`K)7r&O|5rz0%?3YD zZv)+*6esB)WmV-(#ZLd1;4#h8UnTj59E1?xJNImco3Sof?DYTo?zEou|I^T|?rbq} z*;D9Y>~WxER6Hv#dH%l!x?hJomB~?Dk)p*{#MENu|Jx#=>S@v1j@=hK9@+Hud~G3N z{{LwN>g!i)KRf7tb-Zf-iMm0uz1|4-w&@Z=QFDCR2qC@OI;ac6RaxoQe! z3JVo_DQL;xLnqM>N`Mle1SkPYfD)htC;>`<68Ognh@S$W%5<6GyzWLg56;Y4J?hTM zZmhrXE%_$qw#sPTciF1`-~eCYRmKw;T|a^8iuqC#KWu#xaNoU)s2ysFt~Oih{spvf4jtDg*8 zs129=Uv(ek&)O>wb%c|q0JLVhY&ke()$JU(T)3vUMTtEtHc}I&0H7;BGGN1n4?%EB z=ct3ti$qLKngY;@G4u<07k?)aF3x#mzcAtst~R!s_T!SQ>E%RDZf!PTq@08Nxn$p4%G zfZp$ut5&{w+6f8`_H6RK5yeuICjcPkRujJ;?|Bt2FKgt=fg((8W@Rhd_V4GF`wGi= z*U^>x3A7VWs4<2-iC!BvuY@Z+TMOqi>|8VFn-Ti|wV2LomL}3?2SAa%FE_S5dy`<5}*Vq0ZM=ppaduZN`MmhKO`XP|EK={|L_f`<5}*Vq0ZM=ppadv^{{R8%|NjG!X$VSy5}*Vq0ZM=ppaduZN`Mle1SkPY z;QyEa_5c6JZvYKX2~Yx*03|>PPy&Qv1=p(&TQ?c8s1(~~hyM{Xkg!tClG41!^7PU675OWgU$Aa!-(O~;nq#G#7s#;r1 zWDmr50Sq$r>Q+a#{yeHeNYuI0m0w=}`u}&)N%Vsf zpaduZN`Mle1SkPYfD)ht{%Hc@m;ZNVRKE%wi);qKkah!Ggjif=EiNuv{@<>0`F}}S zSyQy^f34;JIcpi!6&iU(59J|cQJmf8RR?f6Vf}vxMon(f!^z_vV92;L!Mml`U~1C( z|47}^Q%f(l0}MJb!)uW59867G{~z7>hnxpa1!j<_S2}q1fz>QEdHsJ6MorDTbCKzM zNNgYMwA%YHt|qMiAIzwJ+?;)4b`%V1Z@g^v;^ml{wEllEqo!5d{$NfRBpxUqaJ_{! zt|qMi@4={klKp%wWiAYMvAG+SG73|Z*8lI0?pxzalULp_IIHD{OV;ag!0~K;qX}&PLVySZvbz|9*_>w>gVGY%GPu z@?QN~ZrX^cN$daHBXzTJ{c;g=+eL%?Lb~H>!utQc5%c{Oqo!|y(S;{J+WP{nPKlf=%~Nn)M!wsY(9dhf#gkvvk|oFvyW;=9(7HXQ}J;{~EIo zAIV6Blz{8I0y3N6YJ&fFMm~6Jd3@u0kj$jy-ESFzsY(9dmr-kiJv5K)2g&>Y36Kx&V%KWO zJxJxs&gv-p9*a%#{}IUl_iH7m#fRk13;SKs-h-)0{vRd!_enSOJI;a;o5T8>{5XlJ zN&epr`QXbhFE8p23)=Q5+WRg5R}=hyGP>_8*E=?Ihfy~@J7zz4%~F&7zZdfV*Yh{W zhr*D;l4r?=7ce!+|D$(II%B@!mXPPy&eX_Q5_hVY83eNt6@B*CfWaJM$P}W;O6Q1Fk!KmN_JBpOii-?{gIJv zRSnX=<+IT zvN1krLbCs&ckHrvaQoemR*|Hczwj=0QX&z}dklL?kOTY4on3`n&ML#BG z1Kb|$hl%lax3)LlgsV;Mq(s?L)g|NS*F^b*JX7xHC(ups;p8%E26wU^w^o7lrU5g{ zt!J_1WdGlfQPa&IzdUCIWJ@ zs+f3bK$_tJ?V@n@^TC`?{r_KlC{r0FKnYL+lmI0_2~Yx*044Ab5fJbHk+(1iX&!p< z1*9qLwJ7{BkmU~Q@&6YX)f-Dj3Qs+R4Bs@vqyCF=Il=!UKWM6~x^84KWbEYKlQX`G zsY(7Hc{0K1+l8*TA;a{$@22nOn40AO(T!^raa=~dgp8JJj=x@^%u4fC<%~pX4dM^{F1~$UfB>#_MhpuiC&`KUszds&2@I@A` zCis7(Zto_wY;q{1woe&*SSuM*ll(ujgTDJl3^P`R%)1MEwH*2#R}{Cdc zvgDavq7o}O+5e+UH`ue@*n2Fbzis9FI==^|Ci#Dat&1@Df$RZD`@-FnXvpq0A=m#= zs&;<7zkL5zn2^2XTl*I50UzrB|Di`V{kD_+ zw{^OIK_4b0k8Lrwtr@N+_>b~J z&3=Qc$^KtjJELc)4ouiFZt2|wwiQJcd~@5zHC%L|u!wgOx#%K+_gFXdj+yC(Bu2@= zgm%Y`J2q~L#Wp4Qe>CPIWjfkKq8p6&-@W_t>1~*rRR2fRBsH?So*xb4J-=xteK@S(6UhHtRIUF<`~Uy)xVnp>bNd=FU~UuVx#{K$$;?*B(F z+UlTxaq}EV4awi{rGs}5ko-R~bOSh#)6~B}x&i0mkIw8l7Z!Xog8xUKj;5ZY^%J8mXU@qzU*Q*{?kubY?A*MQBydczSIKJB$qUf z{o#tMN&Y{FQFCE5TNU(#iE6j`=X&a~vRbH*}Yj>9)g zbE#u1mFi;};x|&MK7($fLbp+Aux(V*JE@4@E!EQ9*RhVh(taIQ7oV#^zfPj*O4GFg z)3urSbhU7DcJ#zI_x*KPm5{E#PNL~b)3w3U75y0?gZ>gQP!+qanrk;>GyDu4+S zDf{ajnz1xv8!Thfbz6wtIn~jzj=u(e9&9N-Q-7X96O|@vgC#1h|Nj+pN;Q-KB|r&K z0+awHKnYL+l)yhlKz#ilmBDR|boX-8AeDdaY54WwtomL(_5U)oiC%j7bjt^jvGVe> z7t4KdIidcKW|Mm=pOuR!fV7VH2W959!_=huKk6Px2)%x6cqpVjidt~Dh`lYEC7)FP zM}pmG@VsDy&D)<(*^1n;6{;!0WjYClneucp6#vU|fW5|b(lbnV@h7#W}xneRF zn^gZ7#pX_*T9OMB?yp-lDHV@Rs{bc2YH{7{@{*^)gkv9e+&ET_#U}Otqhzyt+-^b# zKNNdkg!Q!hn7W?&zh$W1?$4+VezTj4mKD1cZNVq?|D&D@+XX{Dc*j9nx~%(ynFq1h z_0<3EI!Cw8Sqf?D`s)Qc?3F4k_{91DlNdD*x8i=CIgobj^{oSumaO39`agQhn43ew zhvuLyI^2>Yzsg~1()@pvs?L+2A>j9JV4E^Dx%qwWDh{U5bW1YSEDwYWE=7L`03 zuGbceP5b}@MFsetjYzi_@!MI}w%ea>US@wY3|Br&Jb60+8uK>AMO~b}Tv1e72`~Q&-<{z5S zE%XiQMIXEO^pT@j?0Wn^XQx)s6(z`A*2W{~tv*Xl_W$vyAKdPKP_YeUj5)rWx3eRz zCis73$s|7|?7g}MvX?pV^ZTV^YLfp)YUb0W*5j-py=%XLt!8QCYC`|N=nYSvbY|rw zNDEwlRPFcxOil9tD7Hn?`?D6&ke)Je_O#6sxSG)ak7h1Nb>hTTBtXWA1%@ii6)`o* z|D(MJ8ZA1j)cz<;xO?u?G=V0jCi#DqEwUPGiqA%%9UIyNws9DSsY(7Hja+b@4=2X# zhVeaTDaLJJ&s3?$|L3CMEvK$(`5v;RTDAJPbu=qD+5e+tlb_x6t+g6t^yNR<^?|)z z1iAkojUK2x={+j(BBYJnRkA6gj@1Lm{$IAEyTQyAkb2zsVLPMK*p*ZN|JR<$R7D9; z0+awHKnYL+lmI0_3H(z8#QT4=iJl9jbm&|FSzX>f%ev0q7OfutkM{o)uKm7Z#3snu z@iI@^_!`R|ko`aMWPYX61IzC~j#i(JCSRIiYEu0lRdk0Q-Zip67g8sNJsO$E_M+ta zKk}mdt0Bz`T0^#AV%mW%_*`^S{U7bYH=v{GNat~o*5u9W6Om6@S5BV)k36``go#U6 zEr9VZ!Osj&O~uq^mg4LG=)Onjhm36b7}B&H)Qb0A!PNx+j~qO_b9^j4AJSiKH@Nm- zH>M`}e-yjlyQm=N`N+slGfk;u2fupi|E`~hml{8Tv<(l#LLabqY$UJ$FX|qc4{z5t zh4Gfh*1f&O9xWo*{}FS?&*L>KR>C;xbN-``rg0U?{w- zHVy?h61e58L!$?t9v_p2ltNC(djGFscx+PrANj$i+P6L(M{Ut#PQ*-}D9wsZuKyz+ z?6PtF+B^lw{+MQ3G$8?3lk5MgcW0YKDMQ+`%^xZTDq!k*{C|MHm;CxZkiO!K#_gd- zxSHVq(S55tRSr!pgq*?pb3^lbV``HBM~lfhxd#oKXaQNrpN)UEjy>lB?H)j=|06T0 zV&1=hR6b;WYrNB;{1g_ORR0&LJ+`UZUV)rL$;l`AqcAn8{*TmZ1Dl2IL`(3GyrWs} zHWybL*-HP;HUD^Fk^LAn=ORI%AG-umQ{7{;e@_LZ8}^U)toymRz*Rk ze19JWZ}vbJ+5e+4x|Sic?p-a1jJLx+b}L{H_>k-W=*Og8kG;83sL@4I*TQ5;9_!M{ z{r_m}L93JQ^g||)-e+=K)ko~fCUpJ(zxG6?DoTJ7paduZN`Mle1SkPY;GZKP-v6Ts z-HHhhnk&1*B)jz|yBp77xdUY2N&X+r|35YM=@woXWPVWUXD7b`my_rJx6*AjQ`HT! ztj@M5dCA@fkL>@EqjT6Z?2d&HCV1XHJw<;J7MtY%(Tz9JchS`C3F(bTIW~J%M+ZM) z{=aBR_9YgU>ylui!NNxK_T&BF<^=zb8oCu9Yj*XIfbn5wjV*VXu&x}<|0mS{ktOq3 zvB7x?y7KafmU(A2a5cgIBQwb}6<%+94YGpV&fi6VW{6*rC0v*U4mt#384DUUts{hMy)EmhPr2{4N(fa=zIqH?qvUGV( zSyde!iR_Tdb=L!(!+W@Uy9c=VIosF#x_zznDr>$%m#L)7R5sW$mFWf=V%Prl^YsqE z*Z%!=RI65oy1P30g`+$UcXxCS|9y`BulMSYei0X~5oe4Ja0%+WMWcy zQry58tpI0xufP1wVg^Mwz<1HoijGW*3{QxU8bTPM02$&i~sOL!!N+y+dd#f%hNeTOFu5!z#!c~O>C~Zb~7=>bM@Cz zP4T(<>l~V^G*=t!xys>4rt7v6yR5pqw=15lKM%GOpR7Mmp}9(PwZU>VUALw9M0F9= zk*L)N+Y%DB`V^X|G*KHYQR({s|J=we{mzsCB|r&K0+awHKnYL+l)#@SAin;O8reTv z2Td%S0~5vyH`tAJXH^I5ssCSO)OOyQe2+g9X0$0D_4FcpU37B)e;lJO-8V8ZbvdMt z?5<{z#x9GJ>;FjIrNF#sOnb;_XP|w4Rw;Jrr20Q z1R1JR{S2N*U{_9B{~z@pJXok7=rk8HQ^0Lj!6Qsfs{i+8)FpPPy&M*bNlaM{e54L|1W0LcJM;lf8j#zt&Yj7J`Bd?g#LfzWTmv$te%kw8K>PevIcy_ z)a3d9>YbLh=^1PWnIrV3yP96Y)Fl6p+~nm6+bXi!Lq_3S&d8a~S!%NXM}AQLQ_!fy zkuXkW;6u&_Z(L3A|HzAe+;CpF2le20JFGN+^#e>z>iZNAo>?6j$t)HG`>|Urv7cbOTcp{r^-(-EZ&E?6;+myY$+zeY-ZY$||3F4vG2vsN$rH$o>(Iz;(I;F@@c&5dG$wC$UM6I1T$ei4 zg}p^5o&Wz&KDX)jrUWPfN`Mle1SkPYfD)ht{t5x{{vX+3^;XRtV_QJRnpX<%IGHSW zP_O@2E!;nL&+zsz`TLg#ufFNva)SRy^Gegf^p6X??0?%TEYlrv;*>Jfe8#voix@c#&#*2WAO zpHfKk@59(;4ad~9{{L5=$W%iKPy&Nu!XbIsvFUxa;}R zM_pNe-`C^+FEXm9{Wp&2wFstGIK1~=q>sx9{$Diskh9dm0jOeR0XCc`F~Uo zwmvmKs@qq{92L~6oE#BR=FyK8o9zDwGHT`)6Z-oiCwo|R z@aN4-a5bU+j|^So=e_34(ud6Ra*U#XY{QKqp?GhkEkm?^7xeQkm{6ESy?)2ns zOP@h%Mf9f9Z@Vxx(f{W%YOOD{x754|*$0#RIizq|aA{u)btGJ=ra$HUJ|IHt^S?zEW)&KqX_2_MhsjK|Il7cFScZ8S5 zbLPn^o>I(K^iWjbp65>H`f`;NE-B1Z2u3H+4@!U%paduZN`Mle1SkPYfD-t(60lPe zl_aaS4fN*;uQ{AYB}pZLx7T#eGI(h^(?hUxAb1;h7{hlCX1NN{s6(xd|7z_z=m{@7 z4>O)}Tnb!fL}yQX&#tB-wUMpj?|P`;E8*4fFs9tELSXTFgB`p)nEBaV`VM5@Fh2P4 z6}thxa?t_qZ*}cW2M5DT!#0i+UcUpcJ-y!cm1Y;AMS~7@3cqjMuTWT`YldPM2_!X~ zIn&^!d1FR!y8^r>NBVik2qY7Dq+YMIW@#5;+C7YV$_x^X7KOOr0o+0aTMuyNqUc8ByT51;KFNLa%fD{Ji4btPZ3^JIf5>Js=(_&{&1uoG-s0U zZSY$E`R0(x&vCVpU8CREUVHCB&J1{IG%0tya?g5D!mtL!TAQ@g-yc|BHt1tLA9cR|qk3G;O z+C5OT{-2UUn=1eB!jn@xqnNAcqo~Ba#GT0v=Bg=_DJ)dziB6y&lmI0_2~Yx*03|>P zPy&RX3NIiG#j9z1aIsf9}Q{AyxaMXMtb{xPBsV}X0sGI^O z!s&(0KiOewH2I)%9H80@*BS@#L+XozhuJIzi!CO7kDX%g=vLW$P&o`xrLH{;;DFS3 z#~m8+8Z3tIDs(?4$BK>ACbpHM0M)T;jRH6#_5GuLng@Z!iILs&HsV|Oq2`0iL4fMF ztkxiaNPW529IH-Xp&syl(^noAn=}R>dc%vBj4^Kv{OOavk7|~MsZsMm%?$u+jRANe z_3Z-pMzg?DFnyDx+*Mp{WLr4|P<`LEh5&q+X3rGTY&Opai|*b_UW_os)TsHOas;6I zzH5yDi0-@8eVIxdu$Ue8=;H@89AIqS*~p}`vALFs5gM-IlLi1hkh(-a`uH<2eUY+w zyyOpDZDd9Smeg%@0s2VQyZCy_y3}7*?<2| z+=HrY+2_hqll{LtQXdTGtH*+s%A6zSud;D@VxHpncm3+i;=SDR-D%YN4|K}cS za-!4IHMC?^4Gkr-qbm0c6zvrl?&l;34-U8Yu@5&7#t!J!-sysUKHcSl?sCy!yIiDe zw5c;W-NT*SU27)i&jY%3C+E+XX>!u!Y{2AfU1xFzvy=1Z0qwez^XJPnIcah>SaOOs zUDIqYt7>d4ksVX}H@kb~Z+4#=zuJEt=}>ou{(7CJC{57@OVM=Ac6H`w)nA@9^Yhmc zgSzwc*XuMtX?`|bermL>^Y?qz(-3=nng{FGot;0=r^!i^v%!*+*8l(9SO-0u5}*Vq z0ZM=ppaduZN`Mmh7ZVU){~y3CyQC?#blggKQ#AC^?)W3D;$l7Z{~~7Dd8M*7muJBH ztmIESmwdzJgn0n%m}L*OCNI|%G9B_yKOEJaJtvzy4}9h|8Xex7VBjny2I-cOW$`>_QchM`hPgH?4o9u ze$rFnox=#uep7q{KvMl5-FK1gP@f)6;f=v`l^k8XbAeRO*$3IeIlUEtzr1bU%Qt6mIl=#HFwNfGiW6S90zvCV!A@%mFg2Ob>;p@O(x@TQ@wnPV{2BnR zBK7KdF_XbMdQ#gwTf7IKt`- zf#Va+*&{ZUn-7TX{}-t_ZCgD#0M;6dj#&6aVX;a6AHClnBr^^a^akt9F~vi_DBx;B z|GzW3@2?~GFKq=jauxS8ViPbm$^WA{7w@n0r=EHYR?8-L?lG3V&}F^-|L=du-c`H~ z)`v&0Yb{s8icS6hpMM}y1tmZUPy&^!y<%C;0z%Ox(rw(L0yvF-FNMsbfsoHCb}~ zKc4Ap_Q3eV{)bRHbic>&ZtRWlDmNbxJO95w)7NtCPW7xIP`cA;+M;*t)dR@>e-MgY z{QjKkCMaF>5xz}3z`Aep{Qm(ccH0e0KJA9m{KwcaW%pJqj$`r zNOHBl7nDe?k6$rX4O5fq{{zu|&)U*vYXX$sdHhiC`4&t~^8ZL}*8N-4od@Bndspoq z`xfA8g8xVNZ7vXfK*{>CTedGL$J8YMKbYxjZFzpPPG2aA>2>0L>U@@(?Ele~Tc&mx z)vFtns@Ny2cDsP9>H2^F;)gfAHA;XIpaduZN`Mle1SkPY;7<||@Be)YRMZB(m6)py z{N&r$*SD<0z+2dfU;i%~$wwPmxQqm={73qm^X{{}2^u~i`2SV~DotNCd1&DW!>u05 zu6mw}sY&&JZxnm%XqmeU!8Rvr?uMsz?E@fw{Xc)C9^-%X2_L%7cZif*afcO~?Egiv zM=g1&)d#Gme3+tpb0n@d6~F$UDE7#Cm%KS(JIZ9B;%PQD3qHyJ`y%zUK0KwCU}rPP z;q3EGSZp(b{}&+j#*^3DX@PB{6^hDIxDO`z|3IW3eX^gO9Pkgz7_V}&#$uD|{~~H< zw=Q=G2D?5++k&66`#s3@e^IKYwh4~g3$|?(PtEwy7>iBv|04D5dwyNxz)IFz(bT_= z?g4`TcR}jp#YZl#*mo~`oQu}~|K!tEjl5bHLSv`hPT1 z&)DBY<{6x8HuTtoMw>A;ss7&wsmErWu(%Ev4Lki9uflfG`<68Mt@#QT2%)3R`I ztC#ayf_?WM+D8XzvaCrx{yziBL&DC5ECgn9?8D&nvAEpWR`K`F=6i-@{J=gomt4ZMSXzmyPQ>R$eQ1<|#{_lg-BNuR^I)i=83}w}!9Wgb@|9c~Kx}C)I zd0_u>3ZoLMi>nF#-w~-}wbOg#fkVqia>he(HOc>ru6%ySl_hTAkm+&A`r>;mHmUv} zjMV+z%*N`2y-$aN;je9RHNpRTA@!)oe(uY_F}Zo<;-Jfzn&khTk-GKMb03<5y~ctt zoer;AYO? zSyfY0iR{?gn*#-h2ROTgn*`L@-s1Nm`$!RUs6o$JodpD)woq{-QU$*Emua#lSGYb58-10Cy5&Yv&SRl);@uzC{57@OHsQ2-ya(5peIoRlmI0_2~Yx*03|>PPy+u81jN_> z2QVZ3@0=O*?kGH2qN3T)mfezFPyN4)85!aYM~qv*lboW%+Z<(CwMir=)c>_m@XYJ) zGQYz^?xdN5f*#m;a4+UDJd&B0H|}y0rY6<@2cg(^ znqFS^86Li~&~B4899I+S|48jWB&db&J$U48bfddJUPUL>|Iw8PIGqgMdIKJ26m02W zy9|p>n*TqT8R`2Yd6M6Gc(mf~S}EZwOiil)qj$`|d70;+3-CC$c(3}BP)tp#|D)Ie zHl^LYw!@>#+K*OP-Nn^}`oHMMWMKcCH*)aksMX;30V^>zss4{*`%N45%C`WXv>w3t zI&EdC$@Tw%DE732N`;@{an#ZPE4k{{=|id!=vdQ{b@m$u`fM>^0fQ{$ClX(;fGw zT7n}y-PK{60n6E-_5TU;|NW6V*F0&_YH(_kH*Ds0eCh(J|K9_tM^~i2cnJ0d(ZlCF z#hon4|9c{Jbl;rra^P&AVA^s}G3(07{$B)r=!MaK9I&7M%z(d!gQ-dWpO4hj_8lI8 zy2%$_dazM4(RFWL@H2}9Kj;_-R# zB>(S>)bEvKeLTU5879@=XaE+QLu&>Merfz|f6Gt8^C~ae@$@BjQqS&X_ zXLfo8*Mcm4n?@+$YQp+|1DIi9C81{vilM|l{`~EZY(Gd||GytItjEJ$J_jSAq^R?^ zM45CfHfjFN09X?J&)T*B^~wEmyy9c!xGdr2f*pV0ee;?tIxn&kgc?9i|* zx39a=J9eV=gAFAtHQE28TI=gn+e;@~oL#WD879s{h-IF1&b>{eK`*I}T4AG6wY0 z)@w%GyTOW0_WvSmuD+pD_kf<}^M@m_S&`u~1N?b%YM|61tKa$bbS@F!SoQvKf* zsY8`a=Zyk`tnUwwsNkLKB>(S))PaI0ad$y)dz#vZ*J@a7lK*!`>Yx{L?-W3%@Q}Jm zF7Dt-{@)R)-6!x>--6D;{zkV=+3y&6|36Xe2*vV!R-h}`ls{^=K6d3K|L=*^jNjVl z2GD_fw@<>z5KK+-|29b7O|fI?V9*OpaF|rY?sKu=ll;FZwq4Yuqn|)8-m~lGMux1| zWd9FH?c8T;xi#qZYVqmj*vGiqjL!f615ZJE3MD`ZPy&PITAnN4SPbDptLhaFYK=4!-Rbt3z8afl;?}f`XU0lO_3o(S6@J?7ee67=1Y2 ztY~Q+YQp?~8txa!qtTOzdur+9Gv(@1`KV3 zF3nGxjj2ig-vy~3OD3ON1|6PPy&>||26^f0{{*MN|Ec<^qFk|I$k{o$?P7*au=cj09)Sg zuId0^;Z??yDG%@yn68*V8mYHE&!1}ux>t5AxU|*+R}k=ddx@u;lK6KlJsY(97D^l-q+?LY^bnIT-)9J+SCMVDT zcShXQeACNjf^tz7;XcvT^pZ^>m_aN2(U66Xyhu61kK+nO*=Ruqv7Mtk*k$OXK zscq7rYot5-lq0_01F8OxvZeQsonQ8x1HEth!}Pi^&0MI-7aF@#` zF0MA?HrdOO=BRVzl%@Mh9^&@ksB;4f{&h?H&*wDB8Xv zI55yx5aeFRG0WP!SJZI^-M)fuU(sOOSETE9l2x^|l*sO1J4?MPv((8s$kEsPSDMy5 zXkK@o);xtKDoxY|Ow^8bCTi8=l$EG851Q4Ts5MWaiAoc-0TZ=Dor&u1DEht6O4OPM zP3lh6ny1i2rHR^LiOS(fF8$y3=Jf9Wn+c@1?_8&Ws(K*SHc&N=S&$y1l`8rarBBfY zdy3Nj|NrJ^2)#B+fD)htC;>`<5}*Vq0ZQOMN>`A0$jRlmXSrA=7H z#d`Yx$0PZ^#?cSwfx))6iS4zsa5{!!q1ba5bU-U-XV`JleHQEa+|E z+P*l=o~dHNC)NKwkb1+FL6btD6K6|8OAbEwfK>mtM(X@MkDO4S%f@>bOotb+uAE%| zw@2z#>5J0BKzGxFH`-TL;A%Sm|3CUTrNJlxN`Mle1SkPYfD)htD1rZK0^8_+%dyq`zWpY;FZkUZsPcXKb$%W7>CyL2MUn~>N4YmU_Ymft(I8w~8jHW_AK z#?++#f7A~iG4f@Q`B~8a*r%i2r!JV9tn$nSLepN4EBN#Xa#?Q{}+utRK?cY;IwzO7W*9|um3M{@S&PpUkX5f zq=)}74OQ&EN&a8-j-@Q0$1wxlWUJO4?APFGg8vtdT#V^n_yWxq*)eLfa`Qo$n&kgQ z9(?2=iMInlw@AHWL#7O-Ci#C+>>=9g);i=p;z2tf4DQIVN&xv=By8gNK|Duq3_t*L3 z(!u)Bvmdu7vuAXZ*Z=Q^)GK1!&0Y^y_x+Y9B(c|IBm4gVq&}NyG+PBMU;1BFJ&#w> zN&de(Qs=yQyYm@X4xVu{i)t|rv~!;m^PZ;$bJ zurm-Y^-5cesY(97H&Sot+h+L`uvY(3p^mNCK=%KlWEb-X^ee42O^Q~=9db(g&PMZJki_~-PDowEhyD=aZbu*;b66@gJGiw16gXa{}VP5u9W z^l?gqQ38|zB|r&K0+awHKnYL+|J4M<`+t!wirn|4=^C)^wN)6sn_U(~{@>EBa``_= zSy@xG>|d?r|2R94Jo`ucczv*OoO0&fmrpEbL$3d8h=O11cytW#Gm3SleqryyNACX@ zx#$IMNjt}b)vdGp4ix;k`akkyiLrqy4_ASW@%Zf~s;{ssH!`x7|J?^4z!R2SF+u>8 z1RcCQLyjM2{^ry9EHxTEAoTx7BlWNohb69njr&wV9P|^+IZY%kygSV5eoGxiw82Q=_#erEA>xCSlQyVszh| z1*T?)MOQv%(8Zt<;HwSdzgjsLR}P{7nMf zCObGXoOTj&iqbKX`?=kk9N@G=hiZJJtD4EGcJ3^Z9a(#c^{N%9{eG>kUj6T`b+2Dp z{VH9%lCE9ZU~5;V>l)X&RCQIZ1q259y11~?v+AG?DMzbLp@~WpwE+{=sLn*KTK1Qf zs5KAr>rT{~r_e;DiP~_9+TO6vHLI((MX9}Jb&X?Ib*E{KGia95EN!qX75#B4C;F7G zYFg)?rd7|!-+!7`|J$zaG_8J>rYKF(220U&U6VS0<5xWrS-PPy&DTy}Zz()iK?1Ibr^PGo<#M(IW!wi#{gehUIvC_5f-9KL@0a?)dP# zE9f1M-NjvU1&d8u|F0`jn_O0VXvZ(Ru3Tb|GOZySHTrIv~uB%E2`Q3 z{8+Kc{r`?g-651490>-Y1I~WPI)$l8{r`Z}f&*7don)c?;%>Qe#j<|c#Q+B!YAzoNOoxKru;JI5cihj4{j>jH zNudpgcZ8S5bK%J;o>9zI^ifpeUgFN=26NRE$`lqV^h77n4@!U%paduZN`Mle1SkPY zfD-t(6A!QMrd=UCo^JpaE1 zQn!Y$9-}~aQC8zuu54dKp8t=$=sG{^y>ADC{`!pOQ&V!V*d+gNiPTn8oa1ev!xH&@ z4*YyvP4NH7gRgTh2+x`idV80I=)GypQj`6^NUi4;a%(#1Jzk#W7K?jPlK&UIWg)V! z+>=3nhM)7@umxCblK;0rYO{zfJt9DFacPHg>rzZj^8cdP0cXwbmVx%^y^0pcr{ijZ z|F=PE^AQ_YaG}FGvx4oJJ25rM|055+&U@0D^YNe;vwfd!|CP9!?Elx9H!Udp3n+AIH_z>;Gwy{eLe7t$+K2j@}<7KnYL+lmI0_2~Yx*03|>P zkQx9`U3C41_=>cSpu6&5@PpRhSnfhJ0AO3W0H8!23jn4d^#RXMdFh~cdX@L!Ic#r2 z_WvqKz4E!md?zr_wOEs`H4TeRs{gwo_5NL6k%vIX@_K}0Pg6`y^8eOIy;Ji1_<^8( z$$G7D40|U#a{V7w@YnCMux(`nde#g3+?0sRT9t^0mjh{Y!Pe|w~!f6{2qanO6=_029J7*mt{zv#YC4w&ZB z3=Dct-@U#O2UipPzXwv!nfk3w8_>JlOn+hj6iiL>|7g@?{n7qs9h5*{ebd>toJd?v z@c&*&eW3r>{`Wv{-KvbPPy&JG0yc+5eA1>TQ+@9=@R0etV-k%_rh&Qvd&b3)b;=Wus2a0s8xvr z2wE*9Y{7FGVaQ0pAmSjPpyH^As72f;?p3$4)T*^rOD$^ajMhhIC|dFP+j+}Hb_d!OedAyhBdNcN8%f00e*l5C8%|fbjf(*q%GD zuJ>N#?1$PWZQGJ(UXNuiX!}pt|8EiwzZNT+l#QIDZf`r9H8 zvr+a@eq0Yu^8dj&`qZ6An>!$vyg&asr10akP4fTYIC^WpokxZv=K!zZ<8!{|Mw9CQ zarFJT#XmVAm%vTm-gn7kMU(x16pk+UJa9e(wfXzu$2mRXIMF2k?~bD{+AjR44|1Lu zwtLNz&D>~$|EF+twQtgmV&s}uey-BXloL(z{{egMyyp2Wx4Vm6MV}718^f)G7ZdXT z5**E6zHaM$01yBIKmZ5;0U!VbfB+C6<^MbFxoc|W+L7N0 z2}_<`j4euFxeIdsf83tCCKbPIi3mn+-^M!fJl(m`1pnU{M^EN0S$-9{?dW`~wU)D01m>G&<6w8K-TM|HoNXlbdY>L*^jozZ#Ai-jn@IHo5+v!fl&Z z9W+fsu6-~1Z7EsIYMbo;aW2~A)hFS%b|SZPw~dw^ci}{n{6Fq7livpSSUMHC)lbY! zJk+g$X`AH#@qL?I3mMaG9&)bi+tG66X>K&Z|A%90 z&AbYN!;tIwsFZV;+Hj&t{y!MEec@2Usr!)7#^STt8-C$N6Z}7Y<-h$r)8Rgzx$yrX z-|P`5n&kgOarDg-K2_t9aMYfG1?!_Z(Io#*_xHy8Kfc_7g#C)TUl4C$MU(wM-S&2$ z!BMfu#kJGXsoi^UqHFxWIWL4~ecDoFHrM#R-BA3GPIdj}tJ#Ic#b0NyiRGo zYluV<6dRgz+M;cY_E$c0$&S9vDj#Of8CudYc^L24NVBq z>=wwp6{0&x!$V~XdZ01_7@{yl-{BDT(K|$IwzSk4qRd;t`iCeZfFTM)^c@aSZ@ojb zW*aq~AS{E-XU$Jg`lqBUG(lyOR31tFk5Jt)sjY(n2?O$-4M^DkudZ_)paTLx00;m9AOHk_ z01yBIA3OoW>;H!+dAEE7gyG7*0Xy6q5hx3(b5?YIu1s{&D(A5HyvfQP0s&!!O`uUXN-A@+FyE}aiKo< z8GKUyKLSU`-i^4t7`bj7+p7DqLQdPH{C^mZcK_XDw+IObe=}|G$a$P-QvSa)j*ee_ zV6Gc#Kl|+VQ$5+4Z1VH}!8qFb?yQ^Jk@LEipVXfx=d?}A|9j);jz)PMW00%&p=hqk%YMj)od2gkqOoV1O?Zu5_dT^aJCRKd_W%3fzrW!IfdCKy0zd!=00AHX z1c1QX5HRfjv9Y;5&f~`ZD@f>Rw(&)08t@p{P{W`SZM3el#1V@j3G_XN35?(C5d~8DrCz|B{>FCeCJ=MqpxfY&L zj<;p+WJj+5kHXR8=Jh|l9l1>(+q*nDgVi?K{|Djdsl7HmTYg3BMkngm}w2(Io#Lh@(>$nLV9=oOe8&viOz*H=5x8@odrU zoR#gy;R^o#r6uil2{_Rt|4;XK#@F}ST|jQZ3+IgKoymzN`F}j`aeLSni@^(!Fyl{A zmuKu<6UhFbzVh^-C)dA29ojaUbw7x^dH_8C|2AKSpalXz00;m9AOHk_01yBIA36cU z69AuJcj0mVi{bN-aQd_P(buN2oDKQ;|B*Pl&4W)mEk|y=uV>e9&t39Wn5me^jVAbidd@}o@u~XlQ2WyZqon=aIMF2kPxrTP57TxB zP`hs5{n6briW5!p|MZo&EOe(fBOxl{_pW!I6;1a4F*w@o#V^}_M#2u4`?Xrb-Zc^O z{~!9dH{2f(00KY&2mk>f00e*l5O^yBhV6e>DoeKQz|hsL(3%NGTi3T_7gg7i|6fIA z#mt;^Dz`gYb@bTO*7LbpbW;6)Gb*d=H_Pii{{wwznt5Yh9(x@;^7?;r-1aQR=Lepn zb>kYZzfkxi%T?hFK4JgA1bpMOjV;cWqgAV)n3&qJbF$?8e;kz++o&$VN86%>D@@VqjejrybG$D%n&AJt<7kul4}KM+Rg1R` zyS?r0*8fYw{r%MAWBEf@PQIAJOL0wWhIM)J%+fVqNM!sqk`CO zlAQk^O=b0rJaJ^d5S4^~d)RzGdrvlU{@;PhN=aCheYGsnC*(WLr+3b*ai?XuVv ztt{WoD;#o=8%^;4ov5r{VZLT3N2B6<=f2-~*qjqh^8bN!+taJe=Aq)Vsi)V!`kNa~ z@c+R$Iy`vU*~zHzHP8QUN);!XnE%Joa{J8AuTbGPYc^1gUU8!d`G5M#pT(Bm`WqFF zZW32kDd9wu{J$@bo>AXC*98?#y7v3n7TnbXNd6zU-KYKG*+U&s@!*F$r#ETBYMbo; zagQbQqw>=%P{~xgwUNhPa-yOB{{#KnhT8%HKmZ5;0U!VbfB+Bx0{@7BVgDb5y@^kA zr)YCj^lP~E$yoM!aAf~KlFCvH{NZTG4OB2`+%u|FtSSG=>mE2gyRtTRgKO;YftXY|ICp^@Rt zo+_;Oy6BElJy9@9VU)hZQ5wh@rT9COGfLU-aYrdj0izT~>AM@HE&-fPeDMZo+)aGh zZ$@y(DO&;q6$a`%9H^Z*1NF@xc-`Hm*>8q%2P#_v0~H49yBnx&5a;jyn{P&)U#G11 zqPWA9rGQZiqx7APQrQ3RA5Duw5D)+YKmZ5;0U!VbfB+Eq5D6Gg|I=yF-)yg6i$o>4 zpBXoHV;45l>Hk1Joenm(un^;H*p?((b4Q3^SOHukn;aHgP$_V z=)mYw6;zxg4evFkJ2#q;|BuF* z?2fI@eOiD@ZafLQ(=(D2P0Ih{=wzofbH7Sd@}$9@(fmi;XhQxUKXZ}nRM~!CCMxOK zy5B9!WKJ|G|BtC9cS|*Mutvr13!Php&F4lF^8fxg`u3RY<94X9XUD+fphLj0|Bu9vgLm4OwAlp}Mx3Lpw%M@EMJ@h+GL@B-ee@6a2B;|b z*~y)jCfsmx{eMbeK*AbagI_Yay+g(+PBh8?VMSu@Vqgs=vwRldsh7YS$;VxmexPJ zFua@-O{)LLE;=dyc$_wXbbZDUDMicx$O)WL-?48!AsCA3)yaTw;;FJ76eu_S6vfq`|AJK|m zij9}F8OCWFuQh4P%-F=utv=asAAWLSUPAP)b-04Rhvm)W5-BPS7%$rWXe=k1Z8vvzSjvee`G5L(kCw55?VD?!*l-VJMU(x10RD)&ly1pihl*dAZH(yI zo)ZoJU+3En{s9Dl01yBIKmZ5;0U!Vb{}#=;SY0=Ast= zKMUt3=R9B6`yeX*M!LDJ6?;GUTJ!&D=?CJLokqoF^9GGku-8S$>;Dm+|Hm2Kl)~{TlK;o|-M@8%3*#Me^}x;@kNi?t(PaNme?)zz3!TTHLhHYm zf00e*l5C8(0fMNenCv>lDZ{7a@PQe#u z{;=r)%bL{U|8f35(b=`~Rx4DzzpBg#ZQzCz^8b9CgAcBk@i}&Ml?%65q;R82{vW&O zM5mnW3O`hA^rC9(rXM+Nll*@SzVVbkp0ie(?9 zqDlTAm&qg!TK84za2mk>f00e*l5C8%|00;m9AOHk_01)_)34s6q zkb4*I4F~`MAOHk_01yBIKmZ5;0U!VbfItla+W+JI|4i`ye>P?VjQ8*w;~fBbQ(PpR zO;#eJ6yb>?u|g*8s^}(-mPgVXr?KCR=k8v?mcaHEuzkfl+rDCoOQ`;V>fxzpplZDt zXJDXeNnoJDKz(-uRS;s~C>9${?_aZ{>YFdmutd%<)p|>&caUlcV2HvHeRo5&O|Z^} zRqE|~@h)kcO@FoClj;sqJy9@9VU)hZQ5vLol;Ta&IHOeSy%_zYR7(M)6h`Sg8>Kv+ z@su`UIs^61U;L1e5YC@`?f1Is4pco+Fi>HjzO#V}`G1%R0s=q)2mk>f00e*l5C8%| z00;m9An@T6p#4AJHiySQ#GlQN<@4=N+ZWg;+dJ7^wOeV|-%e=zhwbF)38 z>uFcB)0-u3Sa#KXfb-EHUh_{s-g5nwIh8f5I0fI5Zj{*1-%A{5t97|f)fIcJ@#U)8 zwfcn0TI#Z^>Cf{~+1jq91&b$h!aclvZJ5iI?WsQ9?-$mTbT z&HaY8hl1GG-$NW|%`_ZWU0w0}9^9}~^_d+BR*tw~w?##bzd~iFzH2)so6l<4$KO*F zXr&c?rMgnl3-4=JU41V((((~~-KbC66jW9;H>TlWOIA2uRL?8WQY&0(J#u+7>J=~G z)xgFrt#;vXp-c1DcTw5S(ZWwgu~xPdd-=P223XMHMygJhDXPyN+l85ruRis0ianK; z5odm3$Tn2g%W2~+MN?MeVt+3`A9EaSQWG6lTzzKX9NzUg1MuJo-qNYTW>lVWU|!=E zE}TF-ILzq4n!ypLs6O++hIf6I@l{Ow-td?^yU-Tr^lP>!-C2SF*z0{lz|kCR(^q_31k?n6{$&@gKgX z^>OQWDxTfmfXc3Zbhv2d9ZtB1yRWg-G+o;>5$QO@m+W*_w_2uyo@~82o z{QCB%>=)R}?LV@+XtxT#4KE-71b_e#00KY&2mk>f00e*l5U4wWK-&B3o$2_)Z=ER%hwpLtSDPP>?u5$5tn{tu7r+WvOzo1#SM-;W|?m4h?bmMuhmGsO+`NZ%wjZvcgqU7OEKleNz@@ zR95h!wf*|_M_cppf;~ZTf^_Rb?6zP8*X?4pcmfI&U)2PNj7N9&nJqLUwqFBBu@a?9>?ugx9#vK z5^e3i^MT9SU952O1c2@5R8}uur{deQ(AHs3)2HY9v%<*}05*T)aHA1_{`CTF9W|hT zO98tCfIIA$Y!d3V{2>1$NXEek;L1b)!^ITm-1f0Nzyp>*Voq)Wlf=9A&_Vt6aRQ z3P4{PzXz4IML40SSt%+Tm0Q&H#6nivDi==|0kG5f)>PK`8x6!_Pqd}k?5*FO+`|dP zPadf9|22tt8D&0g!hYWMJR>^t%>LnHTc9%c^;L?gthpMo$lpt(T>ucL^XLFB? zJ$jOq7HZ|`v11n;CMnI;%JSYnotjKinyHmdC5qxIW!jV$DcpOf6Y@e+jnXZn`!Z$P zg5!GbNToDUE1Qg1w(7nzZQf%;N@KOMerxf05lLzEMk&4Y)yA_M^pj`u|3=4n)@7FW zE!vv(GFfQf$#%HWF{AzXzTov=Az(>gXr3h2!Z9SoD8IhGJ;vgLsd!l;p z#y4AuX&>sPcmEMR!vFufjOS|16YlAu~Oz z-_Wcf>1m48YysZ=Qqol(6eAOaO2UE?qUC~+gm<BQ1n} ze1HAer4Ie*AvV~L^r7x`=|@@!{rLX+F|rQ*=x+FXl|GbMmwu#$(2wu0A0uk*N2x3* zE2xP@b2pi#aJ;df4KsNPNtiw%|6-h)^L-S^(QPZCq> z$1u#p%kbAJeW+(0I#G2F=*RcgkHOIvjzK|2`Fz74Lz$14d$isUqUuPG+Ivx@gHC*3 zok-{ZZSh)~`t$$z0K9+z5C8%|00;m9AOHk_01yBIK;VNSAhdmxF^f z00e*l5C8)IdICZ_jjOl+>As87XVU}9fU8&gf7zqGpYLd@^!a}KRgO~iTt4IfD~~#! zvNND$)&KjAXc<9LGX7t<&+-*Xsqz1(;}693Cn?qbKW;bgkY~ODr5NueDOCG^?CPER z#TrnmT)o==%bK?P%c2uj4#1Syb065%`!fE&NtmyEwbHl4Z9P17{eS&AkCzWu`fO=l zjbE(!6qQo#|Kr5JPWkO?{nx2-^+Lw~+dcD+A}HznKlpzP0bW1=2mk>f00e*l5C8%| z00;nq|1<$@|8LyzG?nUo=L*ML)@1b=|GzKk^MO=qXKO*N{{MQq-E;#=?Ek&Ap8t27 zb~Toy)cF6?ff?>zhLxT|jsGt%%sG-tQZoMEXpYSp14@?vZ`gg#D+5ZF|8KwR@+>OV ze~BJSEdSsBmmN1rO3m~C_4iy|(wn4Y{Qqy!dtOqhi*mTFbNv4&$yNIANR$7!f%X6X z)1PvxBkD0!~A|Nm07EPD&bzIYNmkk|JN%rmync< z|G(2l!0fB~?;ZRf00e*l5C8%O2x$3#+w6W~ z$}MHMqPqIEt^?=z|6hY_DYt=7Cg79mDp~&jOug#K6R39b<`j3PfaU)ib}q;^a88#0 zKlnwm2T95Jf0K{4=TM@OmwK*~<^LUyTM=>F<^RuTS5N|Je@uzRlkNX)R&JfE|2o0{V|efa0zd!=00AHX1b_e#00KY&2mk>? z1ho9W&Bi&`m80f5Ud38nXUq2g`!_Y|tsFHYj6SKZlI{P6rR~2^j+#GLz*Xw{|B7nK z?fWDplmEX^*x(0}Qp^8epMOL-YEzk>*4h5w)XK)dIa&Vy%D1t1vGNqfozc+s{|>Eo z&-YM{TGmca>nbJV|L?ApJW`ID^sg!_&|3fB`u4W@RIaZJedoGfy~fqssCS`%bRhfDCMwtSjFRhY{rtVe z%=7i;Y5mKNHOl$!eg^4~?hmm6qZH~*iTlG7!e$`x-D zaPCMf_V@B**8kgWx;~>Bl^ZfJMV}I{6v)*7*AMjAZs0mu{{LXs^R`s3PZ-6e#Pa{r zjt4$5aGfmwe{VnwN`LDd|KHGPtA0uv{}1c`;ogQ95C8%|00;m9AOHk_01yBIK;S=3 zKx_R!Yp1J}QZa5-36E!@&($;Q{~hdfuv)2z>qBvrto8rTBz9R^;Rk-OZHS(ztX*<8dtB$|K}<;XFVe+nf$+5 zCN)Tz(n3P_I_I1^{=eSHew7BUljZ+6F6EmWXr1N%hyL7Tq*75x>EX%p|Mxpq9a1Vh z15q`5rvSG9w~A?(uKzkU{@=o6KF|6~%NrIg&AOS)xA(QpG&*dwmqq}u{{jKanu|30 z>A!HXaE)XFdBI&R9D{<4^7+}ReFmpT#ze*|BBB&wi3*vtt0F!~9w}9X`S`de8a~n| zBwi9Emk8uR!O;?dVVyuIc-vP-njkV(E(w>!fBd$gv_3myNc!;X)FGJye1*|(eTtE( zgGZ$QvrxMA%=D~&L$ijYrzuji1@g!kNmqGLj7$(J2@6VymJ31>;^QT;a)stdov%+8 z9~l!ApCpKqBngBWX>M+Ll6VWpkPxH%`h-ptiQU~p^!JYHP--2zQ6+?ad~f|299R2? zF4q$hI&y% zAL?C)Zd3`OAKza;Ce)!Hy}S(eBYh~XF8xRgp&#F0Kgw(GM-SWw27ckF4)v`=KdOY# zkMFA=>HYtp{-12g;3yCP0zd!=00AHX1b_e#00KbZLn5H5|Cc>ZG&}eaHOTYh5`2HG z9Js2gpQ-;pWLzYp27NlxkW!QXKl7<))=+BD<3bu4>zu6m|Gjsz|1{7#>-qm?ZGPH7 z4GP{P;MUc%>i>6-;OA3=L_tcfQm6i3mYaNU#Tsgmx0NBKrv87oxsT&cYT%UEE1V2G zu0Ft(0;cMxBpfFX$(0yrot6Jz>QQmqKqBSThBx*#}Mi|CuSbnl4ZdTs&LJb@jUbzha}jy#q<9 zuK$-+bWB)#(?IL0bE^HnEO$?*+h>#mXNqVNtm{k~`3=fwWsPpDr1Pv%f2Y#wgloT`}u#{WOqxmu+Ej#M)RbpGF*Z>r<}@muf$0zd!= z00AHX1b_e#00KY&2mk>f@WBz#tp6vgSlV;MbZWTM5=!r!g12VimD1K1ui`PH$5%7;|4nUf^;t|(YW#nL4qfAxD5bqu z=pnCaUE}{9s`x)|C!JI6|D~56G~Opx$}ZYW;9RH37w^$P`+vUO44wRcP5r-pFME-_ zrQKz_GJF_bKmZ5;0U!VbfB+Bx0zd!=00AKI&Ikzk>L%h}1@Ox|4|Hodv-O8OLrDpyA$~N)N2T4lJ^Z%!BJ^k)DNvZMw z6+>2wT}euf|3CF?^wZHKrN;j^O{*x4p!zSGq~yAKm0Q;M|E7^a6?3WnOONQGq*7|? z|C>5bZHsvtR%-lzlRr;N50jLd{C@+7HrJ+-lp6nEugm(;9wepO|4T1jE1%a={|HgJ zdd>6y(o0!;BkxiJrbf`?RCk_2RcB>(XCunMoPx&xH+6}5f$!b0Qse)dod3Mal%!<* zfA;GxWh5oz|11AI{H^{wV*CGb{eS(EpoIGWfBi=jE)WO+0U!VbfB+Bx0zd!=00AKI zP6=rF{}=P8{6)#Pte|iMT>tN*RsUZab$l%)KczIJWc+_n!Jo}Y=hX85S1JaOl#Ku1 z*}p8!u+m$|_70!JKU%tEh5kEY`~T_Tn;H_7;Qw(?!wU!i0U!VbfB+Bx0zd!=00AKI zKSY4>|7RU8@6A#Uy|!Qf(+ArA|67MIo{^M{|EDTHI;#kMV)^nDH;DC_8_(ZE3Ne0ySu-qJG1^@Q`ZaY{~{?F|G#GMPg9gb zFRqxt;~BH=U3E^z|KGE3y;nK((GMkDN-8De|CJ|BcGQ1IwEy?is{dc;nmtjOIf