From a9dc6e60af84fe5b237ff4e80f59c09d050e8ad7 Mon Sep 17 00:00:00 2001 From: Todd Roper Date: Thu, 17 Aug 2023 05:54:58 -0700 Subject: [PATCH 1/5] Added flag for simulations and configurations. --- copy_project.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/copy_project.py b/copy_project.py index ef19cd27..d211e1cb 100644 --- a/copy_project.py +++ b/copy_project.py @@ -11,7 +11,7 @@ def extract_args(cli_args) -> tuple: """ Function extracts args from cli command. """ - return cli_args.id, cli_args.source, cli_args.destination + return cli_args.id, cli_args.source, cli_args.destination, cli_args.model_config # @TODO: Add STRATIFY = "Stratify" handling? @@ -34,13 +34,22 @@ def extract_args(cli_args) -> tuple: help="The destination URL to copy the project to including protocol and port " "(if needed). E.g. http://localhost:8001", ) + parser.add_argument( + "-c", + "--model_config", + help="Boolean to determine if the model configurations should be copied.", + default="true", + ) args = parser.parse_args() - project_id, source_url, destination_url = extract_args(args) + project_id, source_url, destination_url, copy_configs = extract_args(args) if project_id and source_url and destination_url: print("Running copy project operation.") copy_class = CopyProject( - pid=project_id, source=source_url, dest=destination_url + pid=project_id, + source=source_url, + dest=destination_url, + copy_configs=copy_configs == "true", ) # Copy the project. copy_class.copy_project_obj() From 37406912923346f34657e8f6b33dead844175cea Mon Sep 17 00:00:00 2001 From: Todd Roper Date: Thu, 17 Aug 2023 05:56:02 -0700 Subject: [PATCH 2/5] Added logic to bypass configs and simulations when flag is false. --- dev/tools/copy_project.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/dev/tools/copy_project.py b/dev/tools/copy_project.py index bd0e95a0..a562f3c3 100644 --- a/dev/tools/copy_project.py +++ b/dev/tools/copy_project.py @@ -88,17 +88,20 @@ class CopyProject: "SimulateCiemssOperation", ] - def __init__(self, pid: int, source: str, dest: str): + def __init__(self, pid: int, source: str, dest: str, copy_configs: bool): self.project_id = pid self.source_url = source self.destination_url = dest + self._copy_model_configurations = copy_configs def copy_project_obj(self): """ Function copies base project to destination. """ self._fetch_project() + print(self.source_project) new_project = scrub_obj(self.source_project) + new_project["assets"] = {} post_url = self.post_url.format(host=self.destination_url, resource="projects") response = post_to_destination(url=post_url, body=new_project) new_project["id"] = response["id"] @@ -183,6 +186,11 @@ def validate_copy(self): ) failed_resources = [] for entity in self.source_project_assets.keys(): + if self._copy_model_configurations is False and entity in [ + "model_configurations", + "simulations", + ]: + continue if len(new_project["assets"][entity]) == len( self.source_project_assets[entity] ): @@ -275,7 +283,10 @@ def _process_workflow_models(self, models: list): raise CopyProjectFailed(message=error_msg) if len(model["outputs"]): for model_output in model["outputs"]: - if model_output["type"] == "modelConfigId": + if ( + model_output["type"] == "modelConfigId" + and self._copy_model_configurations + ): for val in model_output["value"]: self._process_model_config( model_config_id=val, new_model_id=new_model_id @@ -311,7 +322,7 @@ def _process_simulation_node(self, simulation_node: dict): if ( self.id_mapper["simulations"] and simulation_node in self.id_mapper["simulations"].keys() - ): + ) or self._copy_model_configurations is False: return new_simulation_node = simulation_node if len(new_simulation_node["inputs"]): From efeeaaaf2b9f737e2bda325c9628f63ac4e89819 Mon Sep 17 00:00:00 2001 From: Todd Roper Date: Thu, 17 Aug 2023 05:56:39 -0700 Subject: [PATCH 3/5] Fixed project assets return. --- tds/lib/projects.py | 10 ++++++++++ tds/modules/project/controller.py | 17 +++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tds/lib/projects.py b/tds/lib/projects.py index c346d38b..b681e61d 100644 --- a/tds/lib/projects.py +++ b/tds/lib/projects.py @@ -62,3 +62,13 @@ def adjust_project_assets( inactive_ids = set(resource_ids) - set(assets.get(resource_type, [])) for id in inactive_ids: session.delete(session.query(ProjectAsset).get(id)) + + +def clean_up_asset_return(project_assets): + return_obj = {} + for asset in project_assets: + if asset.resource_type not in return_obj: + return_obj[asset.resource_type] = [] + return_obj[asset.resource_type].append(asset.resource_id) + + return return_obj diff --git a/tds/modules/project/controller.py b/tds/modules/project/controller.py index 9ad3cd35..5c4db44f 100644 --- a/tds/modules/project/controller.py +++ b/tds/modules/project/controller.py @@ -10,11 +10,12 @@ from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse from sqlalchemy.engine.base import Engine -from sqlalchemy.orm import Query, Session +from sqlalchemy.orm import Session, joinedload from sqlalchemy.orm.exc import NoResultFound from tds.db import entry_exists, es_client, request_rdb from tds.db.enums import ResourceType +from tds.lib.projects import clean_up_asset_return from tds.modules.project.helpers import ( ResourceDoesNotExist, adjust_project_assets, @@ -97,18 +98,22 @@ def project_get(project_id: int, rdb: Engine = Depends(request_rdb)) -> JSONResp try: if entry_exists(rdb.connect(), Project, project_id): with Session(rdb) as session: - project = session.query(Project).get(project_id) - # pylint: disable-next=unused-variable - parameters: Query[ProjectAsset] = session.query(ProjectAsset).filter( - ProjectAsset.project_id == project_id + project = ( + session.query(Project) + .options(joinedload(Project.assets)) + .get(project_id) ) + project_response = { + **project.__dict__, + "assets": clean_up_asset_return(project.assets), + } else: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) return JSONResponse( status_code=status.HTTP_200_OK, headers={"content-type": "application/json"}, - content=jsonable_encoder(project), + content=jsonable_encoder(project_response), ) except NoResultFound: return JSONResponse( From cb69f85f5ce3ad5977d5cb22c8e35dad33dcf83e Mon Sep 17 00:00:00 2001 From: Todd Roper Date: Thu, 17 Aug 2023 06:26:42 -0700 Subject: [PATCH 4/5] Fixed dataset upload header issue. --- dev/tools/copy_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tools/copy_project.py b/dev/tools/copy_project.py index a562f3c3..86a8e81d 100644 --- a/dev/tools/copy_project.py +++ b/dev/tools/copy_project.py @@ -444,7 +444,7 @@ def _upload_artifact( requests.put( url=upload_url["url"], - files={f"{filename}": download_file.content}, + data=download_file.content, timeout=120, ) From c707e20f8306b3b870b70519b9badd04c6455f5b Mon Sep 17 00:00:00 2001 From: Todd Roper Date: Thu, 17 Aug 2023 11:14:06 -0700 Subject: [PATCH 5/5] Code clean up. --- dev/tools/copy_project.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/tools/copy_project.py b/dev/tools/copy_project.py index 86a8e81d..55a4d74b 100644 --- a/dev/tools/copy_project.py +++ b/dev/tools/copy_project.py @@ -99,7 +99,6 @@ def copy_project_obj(self): Function copies base project to destination. """ self._fetch_project() - print(self.source_project) new_project = scrub_obj(self.source_project) new_project["assets"] = {} post_url = self.post_url.format(host=self.destination_url, resource="projects")