From b8992555e798eda528f24e059a73a29b123b891a Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Fri, 10 Dec 2021 14:21:30 -0500 Subject: [PATCH 01/12] Switching to black. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9a3d7c6..31ae7c0 100644 --- a/Makefile +++ b/Makefile @@ -49,12 +49,12 @@ clean-test: ## remove test and coverage artifacts rm -fr htmlcov/ find . -name '.pytype' -exec rm -fr {} + -lint: ## check style with flake8 +lint: ## check style with black and flake8 + black --extend-exclude '_version.py' --check --diff $(MODULE) tests flake8 $(MODULE) tests - yapf --diff --recursive $(MODULE) tests format: ## reformat with with yapf and isort - yapf --recursive --in-place $(MODULE) tests + black --extend-exclude '_version.py' $(MODULE) tests typing: ## check typing pytype $(MODULE) From 839daf5a855204e34c24d137b35a9e8f6f47fed7 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Fri, 10 Dec 2021 14:21:58 -0500 Subject: [PATCH 02/12] Switching to black. --- setup.cfg | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/setup.cfg b/setup.cfg index a20333d..b518289 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,11 +14,8 @@ omit = [flake8] exclude = docs - -[isort] -lines_after_imports = 0 - -[pytype] +max-line-length = 88 +extend-ignore = E203 [versioneer] # Automatic version numbering scheme @@ -27,11 +24,3 @@ style = pep440 versionfile_source = loop_step/_version.py versionfile_build = loop_step/_version.py tag_prefix = '' - -[yapf] -based_on_style = google -column_limit = 79 -dedent_closing_brackets = True -each_dict_entry_on_separate_line = True -split_before_dot = True -blank_line_before_module_docstring = True From 7438689bef76f83d6cb7e277ba191173516473f5 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Fri, 10 Dec 2021 17:59:00 -0500 Subject: [PATCH 03/12] Running the loop locally in order to catch errors. Also changes directory name to have leading 0's for better listings. --- loop_step/loop.py | 456 ++++++++++++++++++++++++++-------------------- 1 file changed, 255 insertions(+), 201 deletions(-) diff --git a/loop_step/loop.py b/loop_step/loop.py index e7ca8bd..7faa499 100644 --- a/loop_step/loop.py +++ b/loop_step/loop.py @@ -3,9 +3,13 @@ """Non-graphical part of the Loop step in a SEAMM flowchart""" import logging -import os.path +from pathlib import Path +import sys import traceback +import psutil +import pprint + import loop_step import seamm import seamm_util @@ -49,12 +53,24 @@ def version(self): """ return loop_step.__version__ + @property + def iter_format(self): + if self._loop_length is None: + return "07" + else: + n = len(str(self._loop_length)) + return f"0{n}" + @property def git_revision(self): """The git version of this module. """ return loop_step.__git_revision__ + @property + def working_path(self): + return Path(self.directory) / f"iter_{self._loop_value:{self.iter_format}}" + def description_text(self, P=None): """Return a short description of this step. @@ -114,6 +130,9 @@ def describe(self): def run(self): """Run a Loop step. """ + # If the loop is empty, just go on + if self.loop_node() is None: + return self.exit_node() # Set up the directory, etc. super().run() @@ -128,208 +147,211 @@ def run(self): self._file_handler = None job.handlers[0].setLevel(printing.NORMAL) - if P['type'] == 'For': - if self._loop_value is None: - self.logger.info( - 'For {} from {} to {} by {}'.format( - P['variable'], P['start'], P['end'], P['step'] - ) - ) + # Cycle through the iterations, setting up the first time. + next_node = self + while next_node is not None: + if next_node is self: + next_node = self.loop_node() + + if P['type'] == 'For': + if self._loop_value is None: + self.logger.info( + 'For {} from {} to {} by {}'.format( + P['variable'], P['start'], P['end'], P['step'] + ) + ) - self.logger.info('Initializing loop') - self._loop_value = P['start'] - self.set_variable(P['variable'], self._loop_value) - if self.variable_exists('_loop_indices'): + self.logger.info('Initializing loop') + self._loop_value = P['start'] + self.set_variable(P['variable'], self._loop_value) + self._loop_length = len( + range(P["start"], P["end"]+1, P["step"]) + ) + if self.variable_exists('_loop_indices'): + tmp = self.get_variable('_loop_indices') + self.set_variable( + '_loop_indices', (*tmp, self._loop_value) + ) + else: + self.set_variable('_loop_indices', (self._loop_value,)) + self.set_variable('_loop_index', self._loop_value) + else: + self.write_final_structure() + + self._loop_value += P['step'] + self.set_variable(P['variable'], self._loop_value) + + # Set up the index variables + tmp = self.get_variable('_loop_indices') + self.set_variable( + '_loop_indices', ( + *tmp[0:-1], + self._loop_value, + ) + ) + self.set_variable('_loop_index', self._loop_value) + + # See if we are at the end of loop + if self._loop_value > P['end']: + self._loop_value = None + + # Revert the loop index variables to the next outer loop + # if there is one, or remove them. + tmp = self.get_variable('_loop_indices') + + if len(tmp) <= 1: + self.delete_variable('_loop_indices') + self.delete_variable('_loop_index') + else: + self.set_variable('_loop_indices', tmp[0:-1]) + self.set_variable('_loop_index', tmp[-2]) + + self.logger.info( + ( + 'The loop over {} from {} to {} by {}' + ' finished successfully' + ).format( + P['variable'], P['start'], P['end'], P['step'] + ) + ) + break + + self.logger.info(' Loop value = {}'.format(self._loop_value)) + elif P['type'] == 'Foreach': + self.logger.info(f"Foreach {P['variable']} in {P['values']}") + if self._loop_value is None: + self._loop_value = -1 + self._loop_length = len(P['values']) + if self.variable_exists('_loop_indices'): + tmp = self.get_variable('_loop_indices') + self.set_variable('_loop_indices', ( + *tmp, + None, + )) + else: + self.set_variable('_loop_indices', (None,)) + + if self._loop_value >= 0: + self.write_final_structure() + + self._loop_value += 1 + + if self._loop_value >= self._loop_length: + self._loop_value = None + self._loop_length = None + + # Revert the loop index variables to the next outer loop + # if there is one, or remove them. + tmp = self.get_variable('_loop_indices') + if len(tmp) <= 1: + self.delete_variable('_loop_indices') + self.delete_variable('_loop_index') + else: + self.set_variable('_loop_indices', tmp[0:-1]) + self.set_variable('_loop_index', tmp[-2]) + self.logger.info('The loop over value finished successfully') + + # return the next node after the loop + break + + value = P['values'][self._loop_value] + self.set_variable(P['variable'], value) + + # Set up the index variables tmp = self.get_variable('_loop_indices') self.set_variable( - '_loop_indices', (*tmp, self._loop_value) + '_loop_indices', ( + *tmp[0:-1], + self._loop_value, + ) ) - else: - self.set_variable('_loop_indices', (self._loop_value,)) self.set_variable('_loop_index', self._loop_value) - else: - self.write_final_structure() - - self._loop_value += P['step'] - self.set_variable(P['variable'], self._loop_value) - - # Set up the index variables - tmp = self.get_variable('_loop_indices') - self.set_variable( - '_loop_indices', ( - *tmp[0:-1], - self._loop_value, - ) - ) - self.set_variable('_loop_index', self._loop_value) - - # See if we are at the end of loop - if self._loop_value > P['end']: - self._loop_value = None - - # Revert the loop index variables to the next outer loop - # if there is one, or remove them. - tmp = self.get_variable('_loop_indices') - - if len(tmp) <= 1: - self.delete_variable('_loop_indices') - self.delete_variable('_loop_index') - else: - self.set_variable('_loop_indices', tmp[0:-1]) - self.set_variable('_loop_index', tmp[-2]) - - self.logger.info( - ( - 'The loop over {} from {} to {} by {}' + self.logger.info(' Loop value = {}'.format(value)) + elif P['type'] == 'For rows in table': + if self._loop_value is None: + self.table_handle = self.get_variable(P['table']) + self.table = self.table_handle['table'] + self.table_handle['loop index'] = True + + self.logger.info( + 'Initialize loop over {} rows in table {}'.format( + self.table.shape[0], P['table'] + ) + ) + self._loop_value = -1 + self._loop_length = self.table.shape[0] + if self.variable_exists('_loop_indices'): + tmp = self.get_variable('_loop_indices') + self.set_variable('_loop_indices', ( + *tmp, + None, + )) + else: + self.set_variable('_loop_indices', (None,)) + + if self._loop_value >= 0: + self.write_final_structure() + + self._loop_value += 1 + if self._loop_value >= self.table.shape[0]: + self._loop_value = None + + self.delete_variable('_row') + # Revert the loop index variables to the next outer loop + # if there is one, or remove them. + tmp = self.get_variable('_loop_indices') + if len(tmp) <= 1: + self.delete_variable('_loop_indices') + self.delete_variable('_loop_index') + else: + self.set_variable('_loop_indices', tmp[0:-1]) + self.set_variable('_loop_index', tmp[-2]) + + # and the other info in the table handle + self.table_handle['loop index'] = False + + self.table = None + self.table_handle = None + + self.logger.info( + 'The loop over table ' + self.parameters['table'].value + ' finished successfully' - ).format( - P['variable'], P['start'], P['end'], P['step'] ) - ) - return self.exit_node() - self.logger.info(' Loop value = {}'.format(self._loop_value)) - elif P['type'] == 'Foreach': - self.logger.info(f"Foreach {P['variable']} in {P['values']}") - if self._loop_value is None: - self._loop_value = -1 - self._loop_length = len(P['values']) - if self.variable_exists('_loop_indices'): + # return the next node after the loop + break + + # Set up the index variables + self.logger.debug(' _loop_value = {}'.format(self._loop_value)) tmp = self.get_variable('_loop_indices') - self.set_variable('_loop_indices', ( - *tmp, - None, - )) - else: - self.set_variable('_loop_indices', (None,)) - - if self._loop_value >= 0: - self.write_final_structure() - - self._loop_value += 1 - - if self._loop_value >= self._loop_length: - self._loop_value = None - self._loop_length = None - - # Revert the loop index variables to the next outer loop - # if there is one, or remove them. - tmp = self.get_variable('_loop_indices') - if len(tmp) <= 1: - self.delete_variable('_loop_indices') - self.delete_variable('_loop_index') - else: - self.set_variable('_loop_indices', tmp[0:-1]) - self.set_variable('_loop_index', tmp[-2]) - self.logger.info('The loop over value finished successfully') - - # return the next node after the loop - return self.exit_node() - - value = P['values'][self._loop_value] - self.set_variable(P['variable'], value) - - # Set up the index variables - tmp = self.get_variable('_loop_indices') - self.set_variable( - '_loop_indices', ( - *tmp[0:-1], - self._loop_value, - ) - ) - self.set_variable('_loop_index', self._loop_value) - self.logger.info(' Loop value = {}'.format(value)) - elif P['type'] == 'For rows in table': - if self._loop_value is None: - self.table_handle = self.get_variable(P['table']) - self.table = self.table_handle['table'] - self.table_handle['loop index'] = True - - self.logger.info( - 'Initialize loop over {} rows in table {}'.format( - self.table.shape[0], P['table'] + self.logger.debug(' _loop_indices = {}'.format(tmp)) + self.set_variable( + '_loop_indices', + (*tmp[0:-1], self.table.index[self._loop_value]) + ) + self.logger.debug( + ' --> {}'.format(self.get_variable('_loop_indices')) + ) + self.set_variable( + '_loop_index', self.table.index[self._loop_value] + ) + self.table_handle['current index'] = ( + self.table.index[self._loop_value] ) - ) - self._loop_value = -1 - if self.variable_exists('_loop_indices'): - tmp = self.get_variable('_loop_indices') - self.set_variable('_loop_indices', ( - *tmp, - None, - )) - else: - self.set_variable('_loop_indices', (None,)) - - if self._loop_value >= 0: - self.write_final_structure() - - self._loop_value += 1 - if self._loop_value >= self.table.shape[0]: - self._loop_value = None - - self.delete_variable('_row') - # Revert the loop index variables to the next outer loop - # if there is one, or remove them. - tmp = self.get_variable('_loop_indices') - if len(tmp) <= 1: - self.delete_variable('_loop_indices') - self.delete_variable('_loop_index') - else: - self.set_variable('_loop_indices', tmp[0:-1]) - self.set_variable('_loop_index', tmp[-2]) - - # and the other info in the table handle - self.table_handle['loop index'] = False - - self.table = None - self.table_handle = None - - self.logger.info( - 'The loop over table ' + self.parameters['table'].value + - ' finished successfully' - ) - - # return the next node after the loop - return self.exit_node() - - # Set up the index variables - self.logger.debug(' _loop_value = {}'.format(self._loop_value)) - tmp = self.get_variable('_loop_indices') - self.logger.debug(' _loop_indices = {}'.format(tmp)) - self.set_variable( - '_loop_indices', - (*tmp[0:-1], self.table.index[self._loop_value]) - ) - self.logger.debug( - ' --> {}'.format(self.get_variable('_loop_indices')) - ) - self.set_variable( - '_loop_index', self.table.index[self._loop_value] - ) - self.table_handle['current index'] = ( - self.table.index[self._loop_value] - ) - - row = self.table.iloc[self._loop_value] - self.set_variable('_row', row) - self.logger.debug(' _row = {}'.format(row)) - for edge in self.flowchart.edges(self, direction='out'): - if edge.edge_subtype == 'loop': - self.logger.info( - 'Loop, first node of loop is: {}'.format(edge.node2) - ) + row = self.table.iloc[self._loop_value] + self.set_variable('_row', row) + self.logger.debug(' _row = {}'.format(row)) # Direct most output to iteration.out # A handler for the file - iter_dir = os.path.join( - self.directory, f'iter_{self._loop_value}' - ) - os.makedirs(iter_dir, exist_ok=True) + iter_dir = self.working_path + iter_dir.mkdir(parents=True, exist_ok=True) - self._file_handler = logging.FileHandler( - os.path.join(iter_dir, 'iteration.out') - ) + if self._file_handler is not None: + self._file_handler.close() + job.removeHandler(self._file_handler) + self._file_handler = logging.FileHandler(iter_dir / "iteration.out") self._file_handler.setLevel(printing.NORMAL) formatter = logging.Formatter(fmt='{message:s}', style='{') self._file_handler.setFormatter(formatter) @@ -340,12 +362,38 @@ def run(self): # reasonable self.flowchart.reset_visited() self.set_subids( - (*self._id, 'iter_{}'.format(self._loop_value)) + (*self._id, f"iter_{self._loop_value:{self.iter_format}}") ) - return edge.node2 - else: - # No loop body? just go on? - return self.exit_node() + + # Run through the steps in the loop body + try: + next_node = next_node.run() + except DeprecationWarning as e: + print("\nDeprecation warning: " + str(e)) + traceback.print_exc(file=sys.stderr) + traceback.print_exc(file=sys.stdout) + except Exception as e: + print( + f"Caught exception in loop iteration {self._loop_value}: {str(e)}" + ) + next_node = self + + if self.logger.isEnabledFor(logging.DEBUG): + p = psutil.Process() + self.logger.debug(pprint.pformat(p.open_files())) + + self.logger.debug(f"Bottom of loop {next_node}") + + # Return to the normally scheduled step, i.e. fall out of the loop. + + # Remove any redirection of printing. + if self._file_handler is not None: + self._file_handler.close() + job.removeHandler(self._file_handler) + self._file_handler = None + job.handlers[0].setLevel(printing.NORMAL) + + return self.exit_node() def write_final_structure(self): """Write the final structure""" @@ -353,10 +401,7 @@ def write_final_structure(self): configuration = system_db.system.configuration if configuration.n_atoms > 0: # MMCIF file has bonds - filename = os.path.join( - self.directory, f'iter_{self._loop_value}', - 'final_structure.mmcif' - ) + filename = self.working_path / "final_structure.mmcif" text = None try: text = configuration.to_mmcif_text() @@ -373,10 +418,7 @@ def write_final_structure(self): # CIF file has cell if configuration.periodicity == 3: - filename = os.path.join( - self.directory, f'iter_{self._loop_value}', - 'final_structure.cif' - ) + filename = self.working_path / "final_structure.cif" text = None try: text = configuration.to_cif_text() @@ -478,3 +520,15 @@ def exit_node(self): # loop is the last node in the flowchart self.logger.debug('There is no node after the loop') return None + + def loop_node(self): + """The first node in the loop body""" + + for edge in self.flowchart.edges(self, direction='out'): + if edge.edge_subtype == 'loop': + self.logger.debug(f'Loop, first node in loop is: {edge.node2}') + return edge.node2 + + # There is no body of the loop! + self.logger.debug('There is no loop body') + return None From 543162664576842d2cf6348e83ea9f8220603669 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Fri, 10 Dec 2021 17:59:30 -0500 Subject: [PATCH 04/12] Blackened! --- loop_step/__init__.py | 6 +- loop_step/loop.py | 299 +++++++++++++++++------------------ loop_step/loop_parameters.py | 21 ++- loop_step/loop_step.py | 9 +- loop_step/tk_loop.py | 51 +++--- 5 files changed, 183 insertions(+), 203 deletions(-) diff --git a/loop_step/__init__.py b/loop_step/__init__.py index 2cbd4e7..a2f5483 100644 --- a/loop_step/__init__.py +++ b/loop_step/__init__.py @@ -17,8 +17,8 @@ from ._version import get_versions __author__ = """Paul Saxe""" -__email__ = 'psaxe@molssi.org' +__email__ = "psaxe@molssi.org" versions = get_versions() -__version__ = versions['version'] -__git_revision__ = versions['full-revisionid'] +__version__ = versions["version"] +__git_revision__ = versions["full-revisionid"] del get_versions, versions diff --git a/loop_step/loop.py b/loop_step/loop.py index 7faa499..19559d5 100644 --- a/loop_step/loop.py +++ b/loop_step/loop.py @@ -18,18 +18,17 @@ logger = logging.getLogger(__name__) job = printing.getPrinter() -printer = printing.getPrinter('loop') +printer = printing.getPrinter("loop") class Loop(seamm.Node): - def __init__(self, flowchart=None, extension=None): - '''Setup the non-graphical part of the Loop step in a + """Setup the non-graphical part of the Loop step in a SEAMM flowchart. Keyword arguments: - ''' - logger.debug('Creating Loop {}'.format(self)) + """ + logger.debug("Creating Loop {}".format(self)) self.table_handle = None self.table = None @@ -38,10 +37,7 @@ def __init__(self, flowchart=None, extension=None): self._file_handler = None super().__init__( - flowchart=flowchart, - title='Loop', - extension=extension, - logger=logger + flowchart=flowchart, title="Loop", extension=extension, logger=logger ) # This needs to be after initializing subclasses... @@ -49,8 +45,7 @@ def __init__(self, flowchart=None, extension=None): @property def version(self): - """The semantic version of this module. - """ + """The semantic version of this module.""" return loop_step.__version__ @property @@ -63,8 +58,7 @@ def iter_format(self): @property def git_revision(self): - """The git version of this module. - """ + """The git version of this module.""" return loop_step.__git_revision__ @property @@ -86,39 +80,35 @@ def description_text(self, P=None): if not P: P = self.parameters.values_to_dict() - text = '' + text = "" - if P['type'] == 'For': - subtext = 'For {variable} from {start} to {end} by {step}' - elif P['type'] == 'Foreach': - subtext = 'Foreach {variable} in {values}' - elif P['type'] == 'For rows in table': - subtext = 'For rows in table {table}' + if P["type"] == "For": + subtext = "For {variable} from {start} to {end} by {step}" + elif P["type"] == "Foreach": + subtext = "Foreach {variable} in {values}" + elif P["type"] == "For rows in table": + subtext = "For rows in table {table}" else: - subtext = 'Loop type defined by {type}' + subtext = "Loop type defined by {type}" - text += self.header + '\n' + __(subtext, **P, indent=4 * ' ').__str__() - text += '\n\n' + text += self.header + "\n" + __(subtext, **P, indent=4 * " ").__str__() + text += "\n\n" # Print the body of the loop - for edge in self.flowchart.edges(self, direction='out'): - if edge.edge_subtype == 'loop': - self.logger.debug( - 'Loop, first node of loop is: {}'.format(edge.node2) - ) + for edge in self.flowchart.edges(self, direction="out"): + if edge.edge_subtype == "loop": + self.logger.debug("Loop, first node of loop is: {}".format(edge.node2)) next_node = edge.node2 while next_node and not next_node.visited: next_node.visited = True - text += __(next_node.description_text(), - indent=4 * ' ').__str__() - text += '\n' + text += __(next_node.description_text(), indent=4 * " ").__str__() + text += "\n" next_node = next_node.next() return text def describe(self): - """Write out information about what this node will do - """ + """Write out information about what this node will do""" self.visited = True @@ -128,8 +118,7 @@ def describe(self): return self.exit_node() def run(self): - """Run a Loop step. - """ + """Run a Loop step.""" # If the loop is empty, just go on if self.loop_node() is None: return self.exit_node() @@ -153,83 +142,83 @@ def run(self): if next_node is self: next_node = self.loop_node() - if P['type'] == 'For': + if P["type"] == "For": if self._loop_value is None: self.logger.info( - 'For {} from {} to {} by {}'.format( - P['variable'], P['start'], P['end'], P['step'] + "For {} from {} to {} by {}".format( + P["variable"], P["start"], P["end"], P["step"] ) ) - self.logger.info('Initializing loop') - self._loop_value = P['start'] - self.set_variable(P['variable'], self._loop_value) + self.logger.info("Initializing loop") + self._loop_value = P["start"] + self.set_variable(P["variable"], self._loop_value) self._loop_length = len( - range(P["start"], P["end"]+1, P["step"]) + range(P["start"], P["end"] + 1, P["step"]) ) - if self.variable_exists('_loop_indices'): - tmp = self.get_variable('_loop_indices') - self.set_variable( - '_loop_indices', (*tmp, self._loop_value) - ) + if self.variable_exists("_loop_indices"): + tmp = self.get_variable("_loop_indices") + self.set_variable("_loop_indices", (*tmp, self._loop_value)) else: - self.set_variable('_loop_indices', (self._loop_value,)) - self.set_variable('_loop_index', self._loop_value) + self.set_variable("_loop_indices", (self._loop_value,)) + self.set_variable("_loop_index", self._loop_value) else: self.write_final_structure() - self._loop_value += P['step'] - self.set_variable(P['variable'], self._loop_value) + self._loop_value += P["step"] + self.set_variable(P["variable"], self._loop_value) # Set up the index variables - tmp = self.get_variable('_loop_indices') + tmp = self.get_variable("_loop_indices") self.set_variable( - '_loop_indices', ( + "_loop_indices", + ( *tmp[0:-1], self._loop_value, - ) + ), ) - self.set_variable('_loop_index', self._loop_value) + self.set_variable("_loop_index", self._loop_value) # See if we are at the end of loop - if self._loop_value > P['end']: + if self._loop_value > P["end"]: self._loop_value = None # Revert the loop index variables to the next outer loop # if there is one, or remove them. - tmp = self.get_variable('_loop_indices') + tmp = self.get_variable("_loop_indices") if len(tmp) <= 1: - self.delete_variable('_loop_indices') - self.delete_variable('_loop_index') + self.delete_variable("_loop_indices") + self.delete_variable("_loop_index") else: - self.set_variable('_loop_indices', tmp[0:-1]) - self.set_variable('_loop_index', tmp[-2]) + self.set_variable("_loop_indices", tmp[0:-1]) + self.set_variable("_loop_index", tmp[-2]) self.logger.info( ( - 'The loop over {} from {} to {} by {}' - ' finished successfully' - ).format( - P['variable'], P['start'], P['end'], P['step'] - ) + "The loop over {} from {} to {} by {}" + " finished successfully" + ).format(P["variable"], P["start"], P["end"], P["step"]) ) break - self.logger.info(' Loop value = {}'.format(self._loop_value)) - elif P['type'] == 'Foreach': + self.logger.info(" Loop value = {}".format(self._loop_value)) + elif P["type"] == "Foreach": self.logger.info(f"Foreach {P['variable']} in {P['values']}") if self._loop_value is None: self._loop_value = -1 - self._loop_length = len(P['values']) - if self.variable_exists('_loop_indices'): - tmp = self.get_variable('_loop_indices') - self.set_variable('_loop_indices', ( - *tmp, - None, - )) + self._loop_length = len(P["values"]) + if self.variable_exists("_loop_indices"): + tmp = self.get_variable("_loop_indices") + self.set_variable( + "_loop_indices", + ( + *tmp, + None, + ), + ) else: - self.set_variable('_loop_indices', (None,)) + self.set_variable("_loop_indices", (None,)) if self._loop_value >= 0: self.write_final_structure() @@ -242,52 +231,56 @@ def run(self): # Revert the loop index variables to the next outer loop # if there is one, or remove them. - tmp = self.get_variable('_loop_indices') + tmp = self.get_variable("_loop_indices") if len(tmp) <= 1: - self.delete_variable('_loop_indices') - self.delete_variable('_loop_index') + self.delete_variable("_loop_indices") + self.delete_variable("_loop_index") else: - self.set_variable('_loop_indices', tmp[0:-1]) - self.set_variable('_loop_index', tmp[-2]) - self.logger.info('The loop over value finished successfully') + self.set_variable("_loop_indices", tmp[0:-1]) + self.set_variable("_loop_index", tmp[-2]) + self.logger.info("The loop over value finished successfully") # return the next node after the loop break - value = P['values'][self._loop_value] - self.set_variable(P['variable'], value) + value = P["values"][self._loop_value] + self.set_variable(P["variable"], value) # Set up the index variables - tmp = self.get_variable('_loop_indices') + tmp = self.get_variable("_loop_indices") self.set_variable( - '_loop_indices', ( + "_loop_indices", + ( *tmp[0:-1], self._loop_value, - ) + ), ) - self.set_variable('_loop_index', self._loop_value) - self.logger.info(' Loop value = {}'.format(value)) - elif P['type'] == 'For rows in table': + self.set_variable("_loop_index", self._loop_value) + self.logger.info(" Loop value = {}".format(value)) + elif P["type"] == "For rows in table": if self._loop_value is None: - self.table_handle = self.get_variable(P['table']) - self.table = self.table_handle['table'] - self.table_handle['loop index'] = True + self.table_handle = self.get_variable(P["table"]) + self.table = self.table_handle["table"] + self.table_handle["loop index"] = True self.logger.info( - 'Initialize loop over {} rows in table {}'.format( - self.table.shape[0], P['table'] + "Initialize loop over {} rows in table {}".format( + self.table.shape[0], P["table"] ) ) self._loop_value = -1 self._loop_length = self.table.shape[0] - if self.variable_exists('_loop_indices'): - tmp = self.get_variable('_loop_indices') - self.set_variable('_loop_indices', ( - *tmp, - None, - )) + if self.variable_exists("_loop_indices"): + tmp = self.get_variable("_loop_indices") + self.set_variable( + "_loop_indices", + ( + *tmp, + None, + ), + ) else: - self.set_variable('_loop_indices', (None,)) + self.set_variable("_loop_indices", (None,)) if self._loop_value >= 0: self.write_final_structure() @@ -296,52 +289,51 @@ def run(self): if self._loop_value >= self.table.shape[0]: self._loop_value = None - self.delete_variable('_row') + self.delete_variable("_row") # Revert the loop index variables to the next outer loop # if there is one, or remove them. - tmp = self.get_variable('_loop_indices') + tmp = self.get_variable("_loop_indices") if len(tmp) <= 1: - self.delete_variable('_loop_indices') - self.delete_variable('_loop_index') + self.delete_variable("_loop_indices") + self.delete_variable("_loop_index") else: - self.set_variable('_loop_indices', tmp[0:-1]) - self.set_variable('_loop_index', tmp[-2]) + self.set_variable("_loop_indices", tmp[0:-1]) + self.set_variable("_loop_index", tmp[-2]) # and the other info in the table handle - self.table_handle['loop index'] = False + self.table_handle["loop index"] = False self.table = None self.table_handle = None self.logger.info( - 'The loop over table ' + self.parameters['table'].value + - ' finished successfully' + "The loop over table " + + self.parameters["table"].value + + " finished successfully" ) # return the next node after the loop break # Set up the index variables - self.logger.debug(' _loop_value = {}'.format(self._loop_value)) - tmp = self.get_variable('_loop_indices') - self.logger.debug(' _loop_indices = {}'.format(tmp)) + self.logger.debug(" _loop_value = {}".format(self._loop_value)) + tmp = self.get_variable("_loop_indices") + self.logger.debug(" _loop_indices = {}".format(tmp)) self.set_variable( - '_loop_indices', - (*tmp[0:-1], self.table.index[self._loop_value]) + "_loop_indices", + (*tmp[0:-1], self.table.index[self._loop_value]), ) self.logger.debug( - ' --> {}'.format(self.get_variable('_loop_indices')) - ) - self.set_variable( - '_loop_index', self.table.index[self._loop_value] - ) - self.table_handle['current index'] = ( - self.table.index[self._loop_value] + " --> {}".format(self.get_variable("_loop_indices")) ) + self.set_variable("_loop_index", self.table.index[self._loop_value]) + self.table_handle["current index"] = self.table.index[ + self._loop_value + ] row = self.table.iloc[self._loop_value] - self.set_variable('_row', row) - self.logger.debug(' _row = {}'.format(row)) + self.set_variable("_row", row) + self.logger.debug(" _row = {}".format(row)) # Direct most output to iteration.out # A handler for the file @@ -353,7 +345,7 @@ def run(self): job.removeHandler(self._file_handler) self._file_handler = logging.FileHandler(iter_dir / "iteration.out") self._file_handler.setLevel(printing.NORMAL) - formatter = logging.Formatter(fmt='{message:s}', style='{') + formatter = logging.Formatter(fmt="{message:s}", style="{") self._file_handler.setFormatter(formatter) job.addHandler(self._file_handler) @@ -397,7 +389,7 @@ def run(self): def write_final_structure(self): """Write the final structure""" - system_db = self.get_variable('_system_db') + system_db = self.get_variable("_system_db") configuration = system_db.system.configuration if configuration.n_atoms > 0: # MMCIF file has bonds @@ -407,13 +399,13 @@ def write_final_structure(self): text = configuration.to_mmcif_text() except Exception: message = ( - 'Error creating the mmcif file at the end of the loop\n\n' + "Error creating the mmcif file at the end of the loop\n\n" + traceback.format_exc() ) self.logger.critical(message) if text is not None: - with open(filename, 'w') as fd: + with open(filename, "w") as fd: print(text, file=fd) # CIF file has cell @@ -424,13 +416,13 @@ def write_final_structure(self): text = configuration.to_cif_text() except Exception: message = ( - 'Error creating the cif file at the end of the loop' - '\n\n' + traceback.format_exc() + "Error creating the cif file at the end of the loop" + "\n\n" + traceback.format_exc() ) self.logger.critical(message) if text is not None: - with open(filename, 'w') as fd: + with open(filename, "w") as fd: print(text, file=fd) def default_edge_subtype(self): @@ -445,9 +437,9 @@ def default_edge_subtype(self): """ # how many outgoing edges are there? - n_edges = len(self.flowchart.edges(self, direction='out')) + n_edges = len(self.flowchart.edges(self, direction="out")) - self.logger.debug(f'loop.default_edge_subtype, n_edges = {n_edges}') + self.logger.debug(f"loop.default_edge_subtype, n_edges = {n_edges}") if n_edges == 0: return "loop" @@ -457,9 +449,8 @@ def default_edge_subtype(self): return "too many" def create_parser(self): - """Setup the command-line / config file parser - """ - parser_name = 'loop-step' + """Setup the command-line / config file parser""" + parser_name = "loop-step" parser = seamm_util.getParser() # Remember if the parser exists ... this type of step may have been @@ -474,11 +465,9 @@ def create_parser(self): pass # Now need to walk through the steps in the loop... - for edge in self.flowchart.edges(self, direction='out'): - if edge.edge_subtype == 'loop': - self.logger.debug( - 'Loop, first node of loop is: {}'.format(edge.node2) - ) + for edge in self.flowchart.edges(self, direction="out"): + if edge.edge_subtype == "loop": + self.logger.debug("Loop, first node of loop is: {}".format(edge.node2)) next_node = edge.node2 while next_node and next_node != self: next_node = next_node.create_parser() @@ -487,7 +476,7 @@ def create_parser(self): def set_id(self, node_id=()): """Sequentially number the loop subnodes""" - self.logger.debug('Setting ids for loop {}'.format(self)) + self.logger.debug("Setting ids for loop {}".format(self)) if self.visited: return None else: @@ -498,11 +487,9 @@ def set_id(self, node_id=()): def set_subids(self, node_id=()): """Set the ids of the nodes in the loop""" - for edge in self.flowchart.edges(self, direction='out'): - if edge.edge_subtype == 'loop': - self.logger.debug( - 'Loop, first node of loop is: {}'.format(edge.node2) - ) + for edge in self.flowchart.edges(self, direction="out"): + if edge.edge_subtype == "loop": + self.logger.debug("Loop, first node of loop is: {}".format(edge.node2)) next_node = edge.node2 n = 0 while next_node and next_node != self: @@ -512,23 +499,23 @@ def set_subids(self, node_id=()): def exit_node(self): """The next node after the loop, if any""" - for edge in self.flowchart.edges(self, direction='out'): - if edge.edge_subtype == 'exit': - self.logger.debug(f'Loop, node after loop is: {edge.node2}') + for edge in self.flowchart.edges(self, direction="out"): + if edge.edge_subtype == "exit": + self.logger.debug(f"Loop, node after loop is: {edge.node2}") return edge.node2 # loop is the last node in the flowchart - self.logger.debug('There is no node after the loop') + self.logger.debug("There is no node after the loop") return None def loop_node(self): """The first node in the loop body""" - for edge in self.flowchart.edges(self, direction='out'): - if edge.edge_subtype == 'loop': - self.logger.debug(f'Loop, first node in loop is: {edge.node2}') + for edge in self.flowchart.edges(self, direction="out"): + if edge.edge_subtype == "loop": + self.logger.debug(f"Loop, first node in loop is: {edge.node2}") return edge.node2 # There is no body of the loop! - self.logger.debug('There is no loop body') + self.logger.debug("There is no loop body") return None diff --git a/loop_step/loop_parameters.py b/loop_step/loop_parameters.py index e972787..7f37a48 100644 --- a/loop_step/loop_parameters.py +++ b/loop_step/loop_parameters.py @@ -23,7 +23,7 @@ class LoopParameters(seamm.Parameters): ), "format_string": "s", "description": "", - "help_text": ("The type of loop used.") + "help_text": ("The type of loop used."), }, "variable": { "default": "i", @@ -32,7 +32,7 @@ class LoopParameters(seamm.Parameters): "enumeration": tuple(), "format_string": "", "description": "loop variable", - "help_text": ("The name of the loop variable.") + "help_text": ("The name of the loop variable."), }, "start": { "default": "1", @@ -41,7 +41,7 @@ class LoopParameters(seamm.Parameters): "enumeration": tuple(), "format_string": "", "description": "from", - "help_text": ("The starting value of the loop.") + "help_text": ("The starting value of the loop."), }, "end": { "default": "10", @@ -50,7 +50,7 @@ class LoopParameters(seamm.Parameters): "enumeration": tuple(), "format_string": "", "description": "to", - "help_text": ("The ending value of the loop.") + "help_text": ("The ending value of the loop."), }, "step": { "default": "1", @@ -59,7 +59,7 @@ class LoopParameters(seamm.Parameters): "enumeration": tuple(), "format_string": "", "description": "by", - "help_text": ("The step or increment of the loop value.") + "help_text": ("The step or increment of the loop value."), }, "values": { "default": "", @@ -68,7 +68,7 @@ class LoopParameters(seamm.Parameters): "enumeration": tuple(), "format_string": "s", "description": "value in", - "help_text": ("The list of values for the loop.") + "help_text": ("The list of values for the loop."), }, "table": { "default": "table1", @@ -77,7 +77,7 @@ class LoopParameters(seamm.Parameters): "enumeration": tuple(), "format_string": "s", "description": "", - "help_text": ("The table to iterate over.") + "help_text": ("The table to iterate over."), }, "where": { "default": "use all rows", @@ -86,7 +86,7 @@ class LoopParameters(seamm.Parameters): "enumeration": ("use all rows",), "format_string": "s", "description": "where", - "help_text": ("The filter for rows, defaults to all rows.") + "help_text": ("The filter for rows, defaults to all rows."), }, } @@ -94,7 +94,4 @@ def __init__(self, defaults={}, data=None): """Initialize the instance, by default from the default parameters given in the class""" - super().__init__( - defaults={**LoopParameters.parameters, **defaults}, - data=data - ) + super().__init__(defaults={**LoopParameters.parameters, **defaults}, data=data) diff --git a/loop_step/loop_step.py b/loop_step/loop_step.py index 6eb1d1a..ab7117c 100644 --- a/loop_step/loop_step.py +++ b/loop_step/loop_step.py @@ -10,9 +10,9 @@ class LoopStep(object): my_description = { - 'description': 'An interface for Loop', - 'group': 'Control', - 'name': 'Loop' + "description": "An interface for Loop", + "group": "Control", + "name": "Loop", } def __init__(self, flowchart=None, gui=None): @@ -23,8 +23,7 @@ def __init__(self, flowchart=None, gui=None): pass def description(self): - """Return a description of what this extension does - """ + """Return a description of what this extension does""" return LoopStep.my_description def create_node(self, flowchart=None, **kwargs): diff --git a/loop_step/tk_loop.py b/loop_step/tk_loop.py index 6eb731f..48834d0 100644 --- a/loop_step/tk_loop.py +++ b/loop_step/tk_loop.py @@ -26,7 +26,7 @@ def __init__( y=20, w=200, h=50, - my_logger=logger + my_logger=logger, ): """Initialize the graphical Tk Loop node @@ -44,62 +44,59 @@ def __init__( super().__init__( tk_flowchart=tk_flowchart, node=node, - node_type='loop', + node_type="loop", canvas=canvas, x=x, y=y, w=w, h=h, - my_logger=my_logger + my_logger=my_logger, ) def create_dialog(self): """Create the dialog!""" - frame = super().create_dialog(title='Edit Loop Step') + frame = super().create_dialog(title="Edit Loop Step") # Create the widgets and grid them in P = self.node.parameters for key in P: self[key] = P[key].widget(frame) - self['type'].combobox.bind("<>", self.reset_dialog) + self["type"].combobox.bind("<>", self.reset_dialog) def reset_dialog(self, widget=None): """Lay out the edit dialog according to the type of loop.""" # Get the type of loop currently requested - loop_type = self['type'].get() + loop_type = self["type"].get() - logger.debug('Updating edit loop dialog: {}'.format(loop_type)) + logger.debug("Updating edit loop dialog: {}".format(loop_type)) # Remove any widgets previously packed - frame = self['frame'] + frame = self["frame"] for slave in frame.grid_slaves(): slave.grid_forget() # keep track of the row in a variable, so that the layout is flexible # if e.g. rows are skipped to control such as 'method' here row = 0 - self['type'].grid(row=row, column=0, sticky=tk.W) - if loop_type == 'For': - self['variable'].grid(row=row, column=2, sticky=tk.W) - self['start'].grid(row=row, column=3, sticky=tk.W) - self['end'].grid(row=row, column=4, sticky=tk.W) - self['step'].grid(row=row, column=5, sticky=tk.W) - elif loop_type == 'Foreach': - self['variable'].grid(row=row, column=2, sticky=tk.W) - self['values'].grid(row=row, column=3, sticky=tk.W) - elif loop_type == 'For rows in table': - self['table'].grid(row=row, column=2, sticky=tk.W) + self["type"].grid(row=row, column=0, sticky=tk.W) + if loop_type == "For": + self["variable"].grid(row=row, column=2, sticky=tk.W) + self["start"].grid(row=row, column=3, sticky=tk.W) + self["end"].grid(row=row, column=4, sticky=tk.W) + self["step"].grid(row=row, column=5, sticky=tk.W) + elif loop_type == "Foreach": + self["variable"].grid(row=row, column=2, sticky=tk.W) + self["values"].grid(row=row, column=3, sticky=tk.W) + elif loop_type == "For rows in table": + self["table"].grid(row=row, column=2, sticky=tk.W) else: - raise RuntimeError( - "Don't recognize the loop_type {}".format(loop_type) - ) + raise RuntimeError("Don't recognize the loop_type {}".format(loop_type)) row += 1 def right_click(self, event): - """Probably need to add our dialog... - """ + """Probably need to add our dialog...""" super().right_click(event) self.popup_menu.add_command(label="Edit..", command=self.edit) @@ -118,9 +115,9 @@ def default_edge_subtype(self): """ # how many outgoing edges are there? - n_edges = len(self.tk_flowchart.edges(self, direction='out')) + n_edges = len(self.tk_flowchart.edges(self, direction="out")) - logger.debug('loop.default_edge_subtype, n_edges = {}'.format(n_edges)) + logger.debug("loop.default_edge_subtype, n_edges = {}".format(n_edges)) if n_edges == 0: return "loop" @@ -135,7 +132,7 @@ def next_anchor(self): """ # how many outgoing edges are there? - n_edges = len(self.tk_flowchart.edges(self, direction='out')) + n_edges = len(self.tk_flowchart.edges(self, direction="out")) if n_edges == 0: return "e" From 38fcbbfa438bd9a1682525f72b0e9ed6e2b15650 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Sat, 11 Dec 2021 21:29:33 -0500 Subject: [PATCH 05/12] Added query on column of tables, and handling of errors. --- loop_step/loop.py | 171 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 139 insertions(+), 32 deletions(-) diff --git a/loop_step/loop.py b/loop_step/loop.py index 19559d5..b898140 100644 --- a/loop_step/loop.py +++ b/loop_step/loop.py @@ -4,6 +4,7 @@ import logging from pathlib import Path +import re import sys import traceback @@ -134,7 +135,71 @@ def run(self): if self._file_handler is not None: job.removeHandler(self._file_handler) self._file_handler = None - job.handlers[0].setLevel(printing.NORMAL) + + # Find the handler for job.out and set the level up + job_handler = None + out_handler = None + for handler in job.handlers: + if ( + isinstance(handler, logging.FileHandler) + and "job.out" in handler.baseFilename + ): + job_handler = handler + job_level = job_handler.level + job_handler.setLevel(printing.JOB) + elif isinstance(handler, logging.StreamHandler): + out_handler = handler + out_level = out_handler.level + out_handler.setLevel(printing.JOB) + + # Set up some unchanging variables + if P["type"] == "For rows in table": + if self._loop_value is None: + self.table_handle = self.get_variable(P["table"]) + self.table = self.table_handle["table"] + self.table_handle["loop index"] = True + + self.logger.info( + "Initialize loop over {} rows in table {}".format( + self.table.shape[0], P["table"] + ) + ) + self._loop_value = -1 + self._loop_length = self.table.shape[0] + if self.variable_exists("_loop_indices"): + tmp = self.get_variable("_loop_indices") + self.set_variable( + "_loop_indices", + ( + *tmp, + None, + ), + ) + else: + self.set_variable("_loop_indices", (None,)) + where = P["where"] + if where == "Use all rows": + pass + elif where == "Select rows where column": + column = P["query-column"] + op = P["query-op"] + value = P["query-value"] + if self.table.shape[0] > 0: + row = self.table.iloc[0] + tmp = pprint.pformat(row) + self.logger.debug(f"Row is\n{tmp}") + if column not in row: + for key in row.keys(): + if column.lower() == key.lower(): + column = key + break + if column not in row: + raise ValueError( + f"Looping over table with criterion on column '{column}': " + "that column does not exist." + ) + else: + raise NotImplementedError(f"Loop cannot handle '{where}'") # Cycle through the iterations, setting up the first time. next_node = self @@ -258,34 +323,64 @@ def run(self): self.set_variable("_loop_index", self._loop_value) self.logger.info(" Loop value = {}".format(value)) elif P["type"] == "For rows in table": - if self._loop_value is None: - self.table_handle = self.get_variable(P["table"]) - self.table = self.table_handle["table"] - self.table_handle["loop index"] = True - - self.logger.info( - "Initialize loop over {} rows in table {}".format( - self.table.shape[0], P["table"] - ) - ) - self._loop_value = -1 - self._loop_length = self.table.shape[0] - if self.variable_exists("_loop_indices"): - tmp = self.get_variable("_loop_indices") - self.set_variable( - "_loop_indices", - ( - *tmp, - None, - ), - ) - else: - self.set_variable("_loop_indices", (None,)) - if self._loop_value >= 0: self.write_final_structure() - self._loop_value += 1 + # Loop until query is satisfied + while True: + self._loop_value += 1 + + if self._loop_value >= self.table.shape[0]: + break + + if where == "Use all rows": + break + + row = self.table.iloc[self._loop_value] + + self.logger.debug(f"Query {row[column]} {op} {value}") + if op == "==": + if row[column] == value: + break + elif op == "!=": + if row[column] != value: + break + elif op == ">": + if row[column] > value: + break + elif op == ">=": + if row[column] >= value: + break + elif op == "<": + if row[column] < value: + break + elif op == "<=": + if row[column] <= value: + break + elif op == "contains": + if value in row[column]: + break + elif op == "does not contain": + if value not in row[column]: + break + elif op == "contains regexp": + if re.search(value, row[column]) is not None: + break + elif op == "does not contain regexp": + if re.search(value, row[column]) is None: + break + elif op == "is empty": + # Might be numpy.nan, and NaN != NaN hence odd test. + if row[column] == "" or row[column] != row[column]: + break + elif op == "is not empty": + if row[column] != "" and row[column] == row[column]: + break + else: + raise NotImplementedError( + f"Loop query '{op}' not implemented" + ) + if self._loop_value >= self.table.shape[0]: self._loop_value = None @@ -349,7 +444,6 @@ def run(self): self._file_handler.setFormatter(formatter) job.addHandler(self._file_handler) - job.handlers[0].setLevel(printing.JOB) # Add the iteration to the ids so the directory structure is # reasonable self.flowchart.reset_visited() @@ -361,14 +455,19 @@ def run(self): try: next_node = next_node.run() except DeprecationWarning as e: - print("\nDeprecation warning: " + str(e)) + printer.normal("\nDeprecation warning: " + str(e)) traceback.print_exc(file=sys.stderr) traceback.print_exc(file=sys.stdout) except Exception as e: - print( + printer.job( f"Caught exception in loop iteration {self._loop_value}: {str(e)}" ) - next_node = self + if "continue" in P["errors"]: + next_node = self + elif "exit" in P["errors"]: + break + else: + raise if self.logger.isEnabledFor(logging.DEBUG): p = psutil.Process() @@ -383,14 +482,22 @@ def run(self): self._file_handler.close() job.removeHandler(self._file_handler) self._file_handler = None - job.handlers[0].setLevel(printing.NORMAL) + if job_handler is not None: + job_handler.setLevel(job_level) + if out_handler is not None: + out_handler.setLevel(out_level) return self.exit_node() def write_final_structure(self): """Write the final structure""" system_db = self.get_variable("_system_db") - configuration = system_db.system.configuration + system = system_db.system + if system is None: + return + configuration = system.configuration + if configuration is None: + return if configuration.n_atoms > 0: # MMCIF file has bonds filename = self.working_path / "final_structure.mmcif" From 0a00e87205bd749770f1b68790f5e13792f01a02 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Sat, 11 Dec 2021 21:29:46 -0500 Subject: [PATCH 06/12] Added query on column of tables, and handling of errors. --- loop_step/loop_parameters.py | 59 ++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/loop_step/loop_parameters.py b/loop_step/loop_parameters.py index 7f37a48..40e2c53 100644 --- a/loop_step/loop_parameters.py +++ b/loop_step/loop_parameters.py @@ -80,14 +80,67 @@ class LoopParameters(seamm.Parameters): "help_text": ("The table to iterate over."), }, "where": { - "default": "use all rows", + "default": "Use all rows", "kind": "string", "default_units": "", - "enumeration": ("use all rows",), + "enumeration": ("Use all rows", "Select rows where column"), "format_string": "s", - "description": "where", + "description": "", "help_text": ("The filter for rows, defaults to all rows."), }, + "query-column": { + "default": "", + "kind": "string", + "default_units": "", + "enumeration": tuple(), + "format_string": "s", + "description": "", + "help_text": ("The column to test"), + }, + "query-op": { + "default": "==", + "kind": "string", + "default_units": "", + "enumeration": ( + "==", + "!=", + ">", + ">=", + "<", + "<=", + "contains", + "does not contain", + "contains regexp", + "does not contain regexp", + "is empty", + "is not empty", + ), + "format_string": "s", + "description": "", + "help_text": ("The column to test"), + }, + "query-value": { + "default": "", + "kind": "string", + "default_units": "", + "enumeration": tuple(), + "format_string": "s", + "description": "", + "help_text": ("Value to use in the test"), + }, + "errors": { + "default": "continue to next iteration", + "kind": "string", + "default_units": "", + "enumeration": ( + "continue to next iteration", + "exit the loop", + "stop the job", + ), + "format_string": "s", + "description": "On errors", + "help_text": ("How to handle errors"), + }, } def __init__(self, defaults={}, data=None): From 37f652fdce312207ebe524559efef10f9f774be9 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Sat, 11 Dec 2021 21:29:57 -0500 Subject: [PATCH 07/12] Added query on column of tables, and handling of errors. --- loop_step/tk_loop.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/loop_step/tk_loop.py b/loop_step/tk_loop.py index 48834d0..38c742a 100644 --- a/loop_step/tk_loop.py +++ b/loop_step/tk_loop.py @@ -62,7 +62,12 @@ def create_dialog(self): for key in P: self[key] = P[key].widget(frame) - self["type"].combobox.bind("<>", self.reset_dialog) + self["type"].bind("<>", self.reset_dialog) + self["where"].bind("<>", self.reset_dialog) + self["query-op"].bind("<>", self.reset_dialog) + + for widget in ("type", "where", "query-op", "errors"): + self[widget].combobox.config(state="readonly") def reset_dialog(self, widget=None): """Lay out the edit dialog according to the type of loop.""" @@ -80,20 +85,36 @@ def reset_dialog(self, widget=None): # keep track of the row in a variable, so that the layout is flexible # if e.g. rows are skipped to control such as 'method' here row = 0 - self["type"].grid(row=row, column=0, sticky=tk.W) + self["type"].grid(row=row, column=0, columnspan=2, sticky=tk.W) if loop_type == "For": self["variable"].grid(row=row, column=2, sticky=tk.W) self["start"].grid(row=row, column=3, sticky=tk.W) self["end"].grid(row=row, column=4, sticky=tk.W) self["step"].grid(row=row, column=5, sticky=tk.W) elif loop_type == "Foreach": + frame.columnconfigure(5, weight=0) self["variable"].grid(row=row, column=2, sticky=tk.W) - self["values"].grid(row=row, column=3, sticky=tk.W) + self["values"].grid(row=row, column=3, sticky=tk.EW) + frame.columnconfigure(3, weight=1) elif loop_type == "For rows in table": - self["table"].grid(row=row, column=2, sticky=tk.W) + frame.columnconfigure(3, weight=0) + self["table"].grid(row=row, column=2, columnspan=2, sticky=tk.EW) + row += 1 + self["where"].grid(row=row, column=1, columnspan=2, sticky=tk.EW) + where = self["where"].get() + if where != "Use all rows": + self["query-column"].grid(row=row, column=3, sticky=tk.EW) + self["query-op"].grid(row=row, column=4) + op = self["query-op"].get() + if "empty" not in op: + self["query-value"].grid(row=row, column=5, sticky=tk.EW) + frame.columnconfigure(5, weight=1) else: raise RuntimeError("Don't recognize the loop_type {}".format(loop_type)) row += 1 + self["errors"].grid(row=row, column=0, columnspan=4, sticky=tk.W) + row += 1 + frame.columnconfigure(0, minsize=40) def right_click(self, event): """Probably need to add our dialog...""" From 79248069455a6df37f1f2c097c9ccaf781e49a1f Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Sun, 12 Dec 2021 06:11:30 -0500 Subject: [PATCH 08/12] Switched from yapf to black. --- .github/workflows/BranchCI.yaml | 2 +- .github/workflows/CI.yaml | 2 +- .github/workflows/Release.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/BranchCI.yaml b/.github/workflows/BranchCI.yaml index 53b842d..cf0a561 100644 --- a/.github/workflows/BranchCI.yaml +++ b/.github/workflows/BranchCI.yaml @@ -38,8 +38,8 @@ jobs: - name: Run linters shell: bash -l {0} run: | + black --extend-exclude '_version.py' --check --diff loop_step tests flake8 loop_step tests - yapf --diff --recursive loop_step tests - name: Run tests shell: bash -l {0} run: | diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index f690864..493916b 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -46,8 +46,8 @@ jobs: - name: Run linters shell: bash -l {0} run: | + black --extend-exclude '_version.py' --check --diff loop_step tests flake8 loop_step tests - yapf --diff --recursive loop_step tests test: name: Test ${{ matrix.os }} Py${{ matrix.python-version }} diff --git a/.github/workflows/Release.yaml b/.github/workflows/Release.yaml index 626093d..0569a37 100644 --- a/.github/workflows/Release.yaml +++ b/.github/workflows/Release.yaml @@ -39,8 +39,8 @@ jobs: - name: Run linters shell: bash -l {0} run: | + black --extend-exclude '_version.py' --check --diff loop_step tests flake8 loop_step tests - yapf --diff --recursive loop_step tests test: name: Test ${{ matrix.os }} Py${{ matrix.python-version }} From 85e38599943ea4b6f86136c3b4e25e46282e9706 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Sun, 12 Dec 2021 06:21:29 -0500 Subject: [PATCH 09/12] Switched from yapf to black. --- devtools/conda-envs/test_env.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/devtools/conda-envs/test_env.yaml b/devtools/conda-envs/test_env.yaml index b639cc5..bb53772 100644 --- a/devtools/conda-envs/test_env.yaml +++ b/devtools/conda-envs/test_env.yaml @@ -12,6 +12,7 @@ dependencies: - seamm # Testing + - black - codecov - flake8 - pytest @@ -19,9 +20,6 @@ dependencies: # Pip-only installs - pip: - # Testing - - yapf - # Documentation - rinohtype - pygments From 34a3729859ab2f185b618f1c3283f0690f067eca Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Sun, 12 Dec 2021 06:24:33 -0500 Subject: [PATCH 10/12] Cleaned out unused files. --- .readthedocs.yml | 21 ---------- .travis.yml | 99 ------------------------------------------------ .yapfignore | 2 - 3 files changed, 122 deletions(-) delete mode 100644 .readthedocs.yml delete mode 100644 .travis.yml delete mode 100644 .yapfignore diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 0fa71cf..0000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,21 +0,0 @@ -# .readthedocs.yml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: docs/conf.py - -# Optionally build your docs in additional formats such as PDF and ePub -formats: all - -# Optionally set the version of Python and requirements required to build your docs -# python: -# version: 3.7 - -# Use conda environment -conda: - environment: devtools/conda-envs/test_env.yaml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 14cd2c4..0000000 --- a/.travis.yml +++ /dev/null @@ -1,99 +0,0 @@ -language: python -script: pytest -os: linux - -before_install: - # Additional info about the build - - uname -a - - df -h - - ulimit -a - - # Install the Python environment - - source devtools/travis-ci/before_install.sh - - python -V - -install: - # Create test environment for package - - python devtools/scripts/create_conda_env.py -n=test -p=$PYTHON_VER devtools/conda-envs/test_env.yaml - # Activate the test environment - - conda activate test - # Build and install package - #- python setup.py develop --no-deps - - pip install -e . - # Print details of the environment - - conda list - - pip freeze - -notifications: - email: false - -stages: - - lint - - test - - deploy - -jobs: - fast_finish: true - include: - - stage: lint - name: "Check formatting of code for PEP-8" - os: linux - language: generic - env: PYTHON_VER=3.8 - script: - - flake8 loop_step tests - - yapf --diff --recursive loop_step tests - - - stage: test - name: "Tests on MacOS Python 3.7" - script: - - pytest -v tests/ - os: osx - language: generic - env: PYTHON_VER=3.7 - - - stage: test - name: "Tests on MacOS Python 3.8" - script: - - pytest -v tests/ - os: osx - language: generic - env: PYTHON_VER=3.8 - - - stage: test - name: "Tests on Linux Python 3.7" - script: - - pytest -v tests/ - os: linux - language: generic # No need to set Python version since its conda - env: PYTHON_VER=3.7 - - - stage: test - name: "Tests and coverage on Linux Python 3.8" - script: - - pytest -v --cov=loop_step tests/ - after_success: - - codecov - os: linux - language: generic - env: PYTHON_VER=3.8 - - - stage: deploy - name: "Deploy to PyPi" - if: tag IS present - os: linux - dist: xenial - python: 3.8 - script: skip - deploy: - provider: pypi - distributions: sdist bdist_wheel - skip_existing: true - on: - condition: $PYTHON_VER = 3.8 - condition: $TRAVIS_OS_NAME = linux - repo: molssi-seamm/loop_step - tags: true - username: seamm - password: - secure: hjhqNZcsLDwJNlIzbkaOTl4nLX86mgc9xs8IwQlkzHQTrGFKCuL4SV7mDZ0QPyowICyrAB8wU/TIVVltmiVcWtdFtWWLF9P2qHJ0cZgarWtxEvGphYaf+0Xk5FeqUd34c38AEUAvN5CEa7BLSyhrQduqfTogaroOkyN1eVFDdnKTVybg7BLaH8RovITbSGxR3pVVfI1ejb4mIdlEUnugU/2CNzUCxCfUCX2XImGrRCTfrg0EKNYyi/PAXv4b1pT+5oJfy2GivmhelNdI0vLX8FRRkaYLjmgNHoM1wl7CpCJX5aJ2c3O4K8k2UDE4258/ixVFixlk44gesDWqevBUFwaa6lDOYSLBA/WaCNhizCc5Y7QVufXHMj8cCS9dhX9GGO6jJsIpZYuLhonaP7lwPuegpRWDfB/rqwRUY2x020osS4N9nnq8u717K0WSCdFumrDalYSXl/0uX2ayDwaZhzmlp0WBQZH6Hf+aRZTxG6l7ovLTkA+cGMgruIkzEmNnWwtXUDeeLN1RxrJvjYy4i0zncFsjKHogLFdDZLwVR/drHOJfoBXMJ1tDyizoZpEXDPynN+8fOhplIiWejg60KYSYxS0Wv/ZH+uDQE9eTdYBRa6YNycOxn2YmlgS/wvLztW80EoKY9FoJGcPKovW+YI5fKA3miKmVFlbYkowB1No= diff --git a/.yapfignore b/.yapfignore deleted file mode 100644 index 485bb6d..0000000 --- a/.yapfignore +++ /dev/null @@ -1,2 +0,0 @@ -loop_step/_version.py -loop_step/*_parameters.py From 12399df5d437b869250bc351dea403fba33839b6 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Sun, 12 Dec 2021 10:17:55 -0500 Subject: [PATCH 11/12] Added missing psutil requirement and simplifying requirements files --- devtools/conda-envs/test_env.yaml | 1 + requirements.txt | 3 ++- requirements_dev.txt | 4 ++-- requirements_install.txt | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 requirements_install.txt diff --git a/devtools/conda-envs/test_env.yaml b/devtools/conda-envs/test_env.yaml index bb53772..659ad00 100644 --- a/devtools/conda-envs/test_env.yaml +++ b/devtools/conda-envs/test_env.yaml @@ -9,6 +9,7 @@ dependencies: - pip # Dependencies for this module + - psutil - seamm # Testing diff --git a/requirements.txt b/requirements.txt index f2a6725..c706f72 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -seamm==2021.2.2 +psutil +seamm diff --git a/requirements_dev.txt b/requirements_dev.txt index 272fe67..1b38e5e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,6 @@ --r requirements_install.txt +-r requirements.txt +black coverage flake8 pytest @@ -7,4 +8,3 @@ pytest-runner sphinx tox twine -yapf diff --git a/requirements_install.txt b/requirements_install.txt deleted file mode 100644 index 9203c7a..0000000 --- a/requirements_install.txt +++ /dev/null @@ -1 +0,0 @@ -seamm From 048cbb5f183412fcd12b388b5ba7a0b05a9c83b2 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Sun, 12 Dec 2021 10:18:32 -0500 Subject: [PATCH 12/12] Added missing psutil requirement and simplifying requirements files --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7f94ac..29d7e8d 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ with open('HISTORY.rst') as history_file: history = history_file.read() -with open('requirements_install.txt') as fd: +with open('requirements.txt') as fd: requirements = fd.read() setup_requirements = [