diff --git a/src/pzserver/__init__.py b/src/pzserver/__init__.py index 4d3944a..82d03e2 100644 --- a/src/pzserver/__init__.py +++ b/src/pzserver/__init__.py @@ -7,3 +7,4 @@ from .catalog import Catalog, SpeczCatalog, TrainingSet from .core import PzServer +from .pipeline import Pipeline diff --git a/src/pzserver/core.py b/src/pzserver/core.py index 2819823..3c4d3b8 100644 --- a/src/pzserver/core.py +++ b/src/pzserver/core.py @@ -12,7 +12,7 @@ from .catalog import SpeczCatalog, TrainingSet from .communicate import PzRequests -from .process import TSMProcess +from .process import CSCProcess, TSMProcess from .upload import PzUpload, UploadData pd.options.display.max_colwidth = None @@ -463,22 +463,17 @@ def __transform_df(self, dataframe, metadata): return results - # ---- Training Set Maker methods ----# - def combine_specz_catalogs(self, catalog_list, duplicates_critera="smallest flag"): - """_summary_ + def combine_specz_catalogs(self, name): + """ + Make combine specz Args: - catalog_list (_type_): _description_ - duplicates_critera (str, optional): _description_. Defaults to "smallest flag". + name (str): combine specz name - Raises: - NotImplementedError: _description_ + Return: + CSCProcess: CSCProcess object """ - # criteria: smallest flag, smallest error - # newest survey - # show progress bar - # return SpeczCatalog object - raise NotImplementedError + return CSCProcess(name, self.api) def make_training_set(self, name): """ @@ -497,6 +492,7 @@ def wait_processing(self, process): Wait for processing to finish (30 minute tolerance time) Args: + process (TSMProcess or CombSpeczProcess): process object Return: diff --git a/src/pzserver/process.py b/src/pzserver/process.py index 96361e9..3d97828 100644 --- a/src/pzserver/process.py +++ b/src/pzserver/process.py @@ -8,32 +8,32 @@ FONTCOLOREND = "\033[0m" -class TSMProcess: - """Responsible for managing user interactions with TSM process.""" +class Process: + """ Responsible for managing user interactions with process """ # pylint: disable=too-many-instance-attributes # Eight is reasonable in this case. - def __init__(self, name, api): - """TSM process class constructor + def __init__(self, pipeline, name, api): + """Process class constructor Args: - name (str): TSM name + pipeline (str): Pipeline name + name (str): Product name api (PzRequests): PzRequests """ self.__api = api self.name = name self.comment = None - self.pipeline = Pipeline("training_set_maker", self.__api) + self.pipeline = Pipeline(pipeline, self.__api) self.__config = self.pipeline.parameters - self.__release = None - self.__specz = None + self.__inputs = [] self.__process = None self.__upload = None def __available_product_types_by_attribute(self, attr): - """List the product types available for TSM by attribute + """List the product types available by attribute Returns: list: product types available @@ -46,7 +46,7 @@ def __available_product_types_by_attribute(self, attr): return available_inputs def available_product_types(self): - """List the product types available for TSM + """List the product types available Returns: list: product types available @@ -54,7 +54,7 @@ def available_product_types(self): return self.__available_product_types_by_attribute("name") def available_product_types_id(self): - """List the product types id available for TSM + """List the product types id available Returns: list: product types id available @@ -63,7 +63,7 @@ def available_product_types_id(self): @property def output(self): - """TSM output info + """Process output info Returns: dict: output info @@ -79,7 +79,7 @@ def output(self): @property def process(self): - """TSM process info + """Process info Returns: dict: process info @@ -94,53 +94,53 @@ def process(self): } @property - def release(self): - """Gets release info + def inputs(self): + """Gets inputs Returns: - dict: release info + list: inputs """ - return self.__release + return self.__inputs - def set_release(self, release_id=None, name=None): - """Set release + def append_input(self, input_id): + """Append input Args: - release_id (int, optional): release ID. Defaults to None. - name (str, optional): release name. Defaults to None. - - Raises: - ValueError: when neither release_id nor name is informed, the raise is triggered + input_id (int): input id """ - if release_id: - release = self.__api.get("releases", release_id) - elif name: - release = self.__api.get_by_name("releases", name) - else: - raise ValueError(f"{FONTCOLORERR}No release selected{FONTCOLOREND}") - self.__release = release + self.__inputs.append(input_id) + @property - def specz(self): - """Gets specz info + def config(self): + """Gets config Returns: - dict: specz info + dict: config """ - return self.__specz + return self.__config - def set_specz(self, specz_id=None, internal_name=None): - """Set specz + def set_config(self, config): + """Set config Args: - specz_id (int, optional): product ID. Defaults to None. + config (dict): config + """ + self.__config.update(config) + + def get_product(self, product_id=None, internal_name=None): + """_summary_ + + Args: + product_id (int, optional): product ID. Defaults to None. internal_name (str, optional): internal name. Defaults to None. Raises: ValueError: when neither specz_id nor internal_name is informed, the raise is triggered """ - if specz_id: - specz = self.__api.get("products", specz_id) + + if product_id: + specz = self.__api.get("products", product_id) elif internal_name: specz = self.__api.get_by_attribute( "products", "internal_name", internal_name @@ -156,76 +156,59 @@ def set_specz(self, specz_id=None, internal_name=None): f"{FONTCOLORERR}Input is not of the expected type.{FONTCOLOREND}" ) - self.__specz = specz - - @property - def config(self): - """Gets config - - Returns: - dict: config - """ - return self.__config + return specz - def set_config(self, config): - """Set config - - Args: - config (dict): config - """ - self.__config.update(config) - - @property - def summary(self): + def summary(self, extra_info=None): """Summary of what will be executed""" - dn_specz = { - "name": self.specz.get("display_name"), - "internal_name": self.specz.get("internal_name"), - "id": self.specz.get("id"), - } - print('-'*30) - print(f"Training Set Maker: {self.name}") - print(f"Release: {self.release.get('display_name')}") - print(f"Specz: {dn_specz}") + print(f"{self.pipeline.display_name}: {self.name}") print(f"Configuration: {self.config}") + + if isinstance(extra_info, list): + for line in extra_info: + print(line) + if self.output: print(f"Output: {self.output}") + print(f"Status: {self.check_status()}") print('-'*30) def run(self): - """Starts TSM processing - - Raises: - ValueError: Fired when no release is set - ValueError: Fired when no specz is set + """Starts processing Returns: dict: process info """ + if self.__process: print(f"Process has already been initialized: {self.process}") return {} - if not self.release: - raise ValueError(f"{FONTCOLORERR}No release selected{FONTCOLOREND}") - - if not self.specz: - raise ValueError(f"{FONTCOLORERR}No specz selected{FONTCOLOREND}") - data_process = { - "release": self.release.get("id"), "display_name": self.name, "used_config": {"param": self.config}, "pipeline": self.pipeline.pipeline_id, - "inputs": [self.specz.get("id")], + "inputs": self.inputs, } + return self.submit(data_process) - process = self.__api.start_process(data_process) - data_process["id"] = process.get("id") - self.__process = data_process + + def submit(self, data): + """Submit process + + Args: + data (dict): process data + + Returns: + dict: process info + """ + + + process = self.__api.start_process(data) + data["id"] = process.get("id") + self.__process = data self.__upload = self.__api.get("products", process.get("upload")) return self.process @@ -252,3 +235,175 @@ def stop(self): return "No process is running" return self.__api.stop_process(self.__process.get("id")) + + +class TSMProcess(Process): + """Responsible for managing user interactions with TSM process.""" + + # pylint: disable=too-many-instance-attributes + # Eight is reasonable in this case. + + def __init__(self, name, api): + """TSM process class constructor + + Args: + name (str): TSM name + api (PzRequests): PzRequests + """ + + super().__init__("training_set_maker", name, api) + self.__api = api + self.__release = None + self.__specz = None + + @property + def release(self): + """Gets release info + + Returns: + dict: release info + """ + return self.__release + + def set_release(self, release_id=None, name=None): + """Set release + + Args: + release_id (int, optional): release ID. Defaults to None. + name (str, optional): release name. Defaults to None. + + Raises: + ValueError: when neither release_id nor name is informed, the raise is triggered + """ + if release_id: + release = self.__api.get("releases", release_id) + elif name: + release = self.__api.get_by_name("releases", name) + else: + raise ValueError(f"{FONTCOLORERR}No release selected{FONTCOLOREND}") + self.__release = release + + @property + def specz(self): + """Gets specz info + + Returns: + dict: specz info + """ + return self.__specz + + def set_specz(self, specz_id=None, internal_name=None): + """Set specz + + Args: + specz_id (int, optional): product ID. Defaults to None. + internal_name (str, optional): internal name. Defaults to None. + + Raises: + ValueError: when neither specz_id nor internal_name is informed, the raise is triggered + """ + + self.__specz = self.get_product(product_id=specz_id, internal_name=internal_name) + self.append_input(self.specz.get("id")) + + def summary(self, extra_info=None): + """Summary of what will be executed""" + + if not extra_info: + extra_info = [] + + if self.specz: + dn_specz = { + "name": self.specz.get("display_name"), + "internal_name": self.specz.get("internal_name"), + "id": self.specz.get("id"), + } + + extra_info.append(f"Specz: {dn_specz}") + + if self.release: + extra_info.append(f"Release: {self.release}") + + super().summary(extra_info=extra_info) + + + def run(self): + """Starts TSM processing + + Returns: + dict: process info + """ + + if self.process: + print(f"Process has already been initialized: {self.process}") + return {} + + data_process = { + "pipeline": self.pipeline.pipeline_id, + "used_config": {"param": self.config}, + "display_name": self.name, + "release": self.release.get("id"), + "inputs": self.inputs, + } + return self.submit(data_process) + + +class CSCProcess(Process): + """Responsible for managing user interactions with CSC process.""" + + # pylint: disable=too-many-instance-attributes + # Eight is reasonable in this case. + + def __init__(self, name, api): + """CSC process class constructor + + Args: + name (str): CSC name + api (PzRequests): PzRequests + """ + + super().__init__("combine_specz", name, api) + # self.__api = api + self.__catalogs = [] + + @property + def input_catalogs(self): + """Gets input catalogs + + Returns: + list: catalog list + """ + return self.__catalogs + + def append_catalog(self, specz_id=None, internal_name=None): + """Append specz + + Args: + specz_id (int, optional): product ID. Defaults to None. + internal_name (str, optional): internal name. Defaults to None. + + Raises: + ValueError: when neither specz_id nor internal_name is informed, the raise is triggered + """ + + specz = self.get_product(product_id=specz_id, internal_name=internal_name) + + dn_specz = { + "name": specz.get("display_name"), + "internal_name": specz.get("internal_name"), + "id": specz.get("id"), + } + + self.__catalogs.append(dn_specz) + self.append_input(specz.get("id")) + + def summary(self, extra_info=None): + """Summary of what will be executed""" + + if not extra_info: + extra_info = [] + + if self.input_catalogs: + extra_info.append(f"Input catalogs: {self.input_catalogs}") + + super().summary(extra_info=extra_info)