From 2b631ad391fed532d2cf9ec8749622ee3116c619 Mon Sep 17 00:00:00 2001 From: gadorlhiac Date: Thu, 2 May 2024 09:50:44 -0700 Subject: [PATCH] ENH Debug break points. Variable substitution minus string formating. --- lute/execution/debug_utils.py | 59 +++++++++++++++++++++++++++++++++++ lute/io/config.py | 43 +++++++++++++++++-------- lute/tasks/task.py | 2 ++ 3 files changed, 91 insertions(+), 13 deletions(-) create mode 100644 lute/execution/debug_utils.py diff --git a/lute/execution/debug_utils.py b/lute/execution/debug_utils.py new file mode 100644 index 00000000..7051a7f1 --- /dev/null +++ b/lute/execution/debug_utils.py @@ -0,0 +1,59 @@ +"""Functions to assist in debugging execution of LUTE. + +Functions: + LUTE_DEBUG_EXIT(env_var: str, str_dump: Optional[str]): Exits the program if + the provided `env_var` is set. Optionally, also prints a message if + provided. + +Exceptions: + ValidationError: Error raised by pydantic during data validation. (From + Pydantic) +""" + +__all__ = ["LUTE_DEBUG_EXIT"] +__author__ = "Gabriel Dorlhiac" + +import os +import sys +import types +from typing import Optional + + +def _stack_inspect(msg: str, str_dump: Optional[str] = None) -> None: + import inspect + + curr_frame: Optional[types.FrameType] = inspect.currentframe() + frame: Optional[types.FrameType] + if curr_frame: + frame = curr_frame.f_back + if frame: + frame = frame.f_back # Go back two stack frames... + else: + frame = None + if frame: + file_name: str = frame.f_code.co_filename + line_no: int = frame.f_lineno + msg = f"{msg} {file_name}, line: {line_no}" + else: + msg = f"{msg} Stack frame not retrievable..." + if str_dump is not None: + msg = f"{msg}\n{str_dump}" + + print(msg, flush=True) + + +def LUTE_DEBUG_EXIT(env_var: str, str_dump: Optional[str] = None) -> None: + if os.getenv(env_var, None): + msg: str = "LUTE_DEBUG_EXIT -" + _stack_inspect(msg, str_dump) + sys.exit(0) + + +def LUTE_DEBUG_PAUSE(env_var: str, str_dump: Optional[str] = None) -> None: + # Need custom signal handlers to implement resume + if os.getenv(env_var, None): + import signal + + msg: str = "LUTE_DEBUG_PAUSE -" + _stack_inspect(msg, str_dump) + signal.pause() diff --git a/lute/io/config.py b/lute/io/config.py index 6afe2972..300cdd01 100644 --- a/lute/io/config.py +++ b/lute/io/config.py @@ -18,7 +18,7 @@ import warnings from typing import List, Dict, Iterator, Dict, Any -import yaml +import pprint import yaml from pydantic import ( BaseModel, @@ -34,10 +34,11 @@ from pydantic.dataclasses import dataclass from .models import * +from lute.execution.debug_utils import LUTE_DEBUG_EXIT def substitute_variables( - config: Dict[str, Any], curr_key: Optional[str] = None + header: Dict[str, Any], config: Dict[str, Any], curr_key: Optional[str] = None ) -> None: """Performs variable substitutions on a dictionary read from config YAML file. @@ -74,30 +75,45 @@ def substitute_variables( have been made. May be identical to the input if no substitutions are needed. """ - _sub_pattern = "\{\{.*\}\}" - iterable: Dict[str, Any] + _sub_pattern = r"\{\{.*\}\}" + iterable: Dict[str, Any] = config if curr_key is not None: # Need to handle nested levels by interpreting curr_key - iterable = config[curr_key] + keys_by_level: List[str] = curr_key.split(".") + for key in keys_by_level: + iterable = iterable[key] else: - iterable = config + ... + # iterable = config for param, value in iterable.items(): if isinstance(value, dict): - substitute_variables(config, curr_key=param) + new_key: str + if curr_key is None: + new_key = param + else: + new_key = f"{curr_key}.{param}" + substitute_variables(header, config, curr_key=new_key) elif isinstance(value, list): ... # Scalars str - we skip numeric types elif isinstance(value, str): matches: List[str] = re.findall(_sub_pattern, value) for m in matches: - key_to_sub: str = m[2:-2].strip() + key_to_sub_maybe_with_fmt: List[str] = m[2:-2].strip().split(":") + key_to_sub: str = key_to_sub_maybe_with_fmt[0] + fmt: Optional[str] = None + if len(key_to_sub_maybe_with_fmt) == 2: + fmt = key_to_sub_maybe_with_fmt[1] sub: str if key_to_sub[0] == "$": sub = os.environ.get(key_to_sub[1:], "") else: - sub = config[key_to_sub] - pattern: str = m.replace("{{", "\{\{").replace("}}", "\}\}") - iterable[param] = re.sub(pattern, sub, value) + try: + sub = config[key_to_sub] + except KeyError: + sub = header[key_to_sub] + pattern: str = m.replace("{{", r"\{\{").replace("}}", r"\}\}") + iterable[param] = re.sub(pattern, str(sub), value) def parse_config(task_name: str = "test", config_path: str = "") -> TaskParameters: @@ -123,7 +139,9 @@ def parse_config(task_name: str = "test", config_path: str = "") -> TaskParamete docs: Iterator[Dict[str, Any]] = yaml.load_all(stream=f, Loader=yaml.FullLoader) header: Dict[str, Any] = next(docs) config: Dict[str, Any] = next(docs) - + # pprint.pprint(config) + substitute_variables(header, config) + LUTE_DEBUG_EXIT("LUTE_DEBUG_EXIT_AT_YAML", pprint.pformat(config)) lute_config: Dict[str, AnalysisHeader] = {"lute_config": AnalysisHeader(**header)} try: task_config: Dict[str, Any] = dict(config[task_name]) @@ -136,5 +154,4 @@ def parse_config(task_name: str = "test", config_path: str = "") -> TaskParamete ) ) parsed_parameters: TaskParameters = globals()[task_config_name](**lute_config) - return parsed_parameters diff --git a/lute/tasks/task.py b/lute/tasks/task.py index d103c72b..5b132623 100644 --- a/lute/tasks/task.py +++ b/lute/tasks/task.py @@ -23,6 +23,7 @@ AnalysisHeader, ) from ..execution.ipc import * +from ..execution.debug_utils import LUTE_DEBUG_EXIT from .dataclasses import * if __debug__: @@ -344,6 +345,7 @@ def _run(self) -> None: time.sleep(0.1) msg: Message = Message(contents=self._formatted_command()) self._report_to_executor(msg) + LUTE_DEBUG_EXIT("LUTE_DEBUG_BEFORE_TPP_EXEC") os.execvp(file=self._cmd, args=self._args_list) def _formatted_command(self) -> str: