From 343100aeedfa6495a0c406b6f063c2949c38178c Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 21 Apr 2023 13:46:54 -0400 Subject: [PATCH 01/24] changes added utils test cases --- pygeoweaver/sc_run.py | 1 + pygeoweaver/utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pygeoweaver/sc_run.py b/pygeoweaver/sc_run.py index 1a041b9..3d6f101 100644 --- a/pygeoweaver/sc_run.py +++ b/pygeoweaver/sc_run.py @@ -1,3 +1,4 @@ +import os import subprocess from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_root_dir diff --git a/pygeoweaver/utils.py b/pygeoweaver/utils.py index afc28ba..fc04892 100644 --- a/pygeoweaver/utils.py +++ b/pygeoweaver/utils.py @@ -45,7 +45,7 @@ def download_geoweaver_jar(overwrite=False): raise RuntimeError("Fail to download geoweaver.jar") -def checkOS(): +def check_os(): if platform.system() == "Linux" or platform == "Linux2": return 1 elif platform.system() == "Darwin": From 98746bbcd19be7de30e95458d469374705404f26 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Wed, 7 Jun 2023 15:06:37 -0400 Subject: [PATCH 02/24] changes to history and run --- pygeoweaver/sc_history.py | 28 ++++++++++++++++++- pygeoweaver/sc_run.py | 58 +++++++++++++++++++++++---------------- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/pygeoweaver/sc_history.py b/pygeoweaver/sc_history.py index 06a875f..788efa3 100644 --- a/pygeoweaver/sc_history.py +++ b/pygeoweaver/sc_history.py @@ -1,4 +1,7 @@ import subprocess +from constants import GEOWEAVER_DEFAULT_ENDPOINT_URL +import requests + from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir @@ -9,4 +12,27 @@ def show_history(history_id): if not history_id: raise RuntimeError("history id is missing") download_geoweaver_jar() - subprocess.run([get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "history", history_id], cwd=f"{get_root_dir()}/") + subprocess.run([get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "history", history_id], + cwd=f"{get_root_dir()}/") + + +def get_process_history(process_id): + """ + Get list of history for a process using process id + :param process_id: str :type process_id: str + """ + if not process_id: + raise Exception("please pass `process_id` as a parameter to the function.") + r = requests.post(f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'process', 'id': process_id}).json() + return r + + +def get_workflow_history(workflow_id): + """ + Get list of history for a workflow using workflow id + :param workflow_id: str + """ + if not workflow_id: + raise Exception("please pass `workflow_id` as a parameter to the function.") + r = requests.get(f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'workflow', 'id': workflow_id}).json() + return r diff --git a/pygeoweaver/sc_run.py b/pygeoweaver/sc_run.py index daf9b23..a992dd0 100644 --- a/pygeoweaver/sc_run.py +++ b/pygeoweaver/sc_run.py @@ -3,7 +3,20 @@ from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir -def run_process(*, process_id: str, host_id: str, password: str, environment: str=None): +def create_process(): + pass + + +def create_workflow(): + """ + Create workflow from workflow.json + :return: + :rtype: + """ + pass + + +def run_process(*, process_id: str, host_id: str, password: str, environment: str = None): """ Run a process @@ -17,8 +30,9 @@ def run_process(*, process_id: str, host_id: str, password: str, environment: st f"--password={password}", f"--environment={environment}", process_id], cwd=f"{get_root_dir()}/") -def run_worklfow(*, workflow_id: str, workflow_folder_path: str=None, workflow_zip_file_path: str=None, - environment_list: str=None, host_list: str, password_list: str): + +def run_workflow(*, workflow_id: str, workflow_folder_path: str = None, workflow_zip_file_path: str = None, + environment_list: str = None, host_list: str = None, password_list: str = None): """ Usage:
run workflow [-d=] [-f=] [-e=]... @@ -36,30 +50,28 @@ def run_worklfow(*, workflow_id: str, workflow_folder_path: str=None, workflow_z download_geoweaver_jar() if not workflow_id and not workflow_folder_path and not workflow_zip_file_path: - raise RuntimeError("Please provide at least one of the three options: workflow id, " \ + raise RuntimeError("Please provide at least one of the three options: workflow id, " "folder path or zip path") - + if workflow_id and not workflow_folder_path and not workflow_zip_file_path: - subprocess.run([get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "run", "workflow", workflow_id, - "-e", environment_list, - "-h", host_list, - "-p", password_list], - cwd=f"{get_root_dir()}/") + command = [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "run", "workflow", workflow_id] + if environment_list: + command.extend(["-e", environment_list]) + command.extend(["-h", host_list, "-p", password_list]) + subprocess.run(command, cwd=f"{get_root_dir()}/") if workflow_folder_path and not workflow_zip_file_path: # command to run workflow from folder - subprocess.run([get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "run", "workflow", workflow_id, - "-d", workflow_folder_path, - "-e", environment_list, - "-h", host_list, - "-p", password_list], - cwd=f"{get_root_dir()}/") + command = [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "run", "workflow", workflow_id] + if environment_list: + command.extend(["-e", environment_list]) + command.extend(["-d", workflow_folder_path, "-h", host_list, "-p", password_list]) + subprocess.run(command, cwd=f"{get_root_dir()}/") if not workflow_folder_path and workflow_zip_file_path: - subprocess.run([get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "run", "workflow", workflow_id, - "-e", environment_list, - "-f", workflow_zip_file_path, - "-h", host_list, - "-p", password_list], - cwd=f"{get_root_dir()}/") - + command = [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "run", "workflow", workflow_id] + if environment_list: + command.extend(["-e", environment_list]) + command.extend(["-f", workflow_zip_file_path, "-h", host_list, "-p", password_list]) + subprocess.run(command, cwd=f"{get_root_dir()}/") + From 6fb7cd367ce38336fe2cf7a3c2c5452a4a78b702 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Wed, 7 Jun 2023 15:22:50 -0400 Subject: [PATCH 03/24] add constants to imports --- pygeoweaver/sc_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index fe11585..96b5a86 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -11,3 +11,4 @@ from pygeoweaver.server import * from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * +from pygeoweaver.constants import * \ No newline at end of file From add43b5609b357d46f35d12963835d0f90117477 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Wed, 7 Jun 2023 15:25:17 -0400 Subject: [PATCH 04/24] import changes --- pygeoweaver/sc_history.py | 2 +- pygeoweaver/sc_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoweaver/sc_history.py b/pygeoweaver/sc_history.py index 788efa3..25528c5 100644 --- a/pygeoweaver/sc_history.py +++ b/pygeoweaver/sc_history.py @@ -1,7 +1,7 @@ import subprocess -from constants import GEOWEAVER_DEFAULT_ENDPOINT_URL import requests +from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index 96b5a86..f00b17d 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -11,4 +11,4 @@ from pygeoweaver.server import * from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * -from pygeoweaver.constants import * \ No newline at end of file +from pygeoweaver.constants import * From dca2ad1a7c6a8bd9f578a91c69e456190189c288 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Wed, 7 Jun 2023 15:26:22 -0400 Subject: [PATCH 05/24] import changes --- pygeoweaver/sc_interface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index f00b17d..fe11585 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -11,4 +11,3 @@ from pygeoweaver.server import * from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * -from pygeoweaver.constants import * From 7d9b3c436ac1e5b495c373b6f4acf57d19b1d5b5 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Wed, 7 Jun 2023 15:41:27 -0400 Subject: [PATCH 06/24] import fix --- pygeoweaver/__main__.py | 2 +- pygeoweaver/sc_history.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pygeoweaver/__main__.py b/pygeoweaver/__main__.py index 78f3b4b..14a7efa 100644 --- a/pygeoweaver/__main__.py +++ b/pygeoweaver/__main__.py @@ -4,7 +4,7 @@ """ from pygeoweaver import detail_host, detail_process, detail_workflow, export_workflow, \ show_history, import_workflow, list_hosts, list_processes, list_workflows, \ - start, stop, reset_password, run_process, run_worklfow, helpwith + start, stop, reset_password, run_process, run_workflow, helpwith from pygeoweaver.server import show diff --git a/pygeoweaver/sc_history.py b/pygeoweaver/sc_history.py index 25528c5..8e3b0e3 100644 --- a/pygeoweaver/sc_history.py +++ b/pygeoweaver/sc_history.py @@ -1,7 +1,7 @@ import subprocess import requests -from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL +from . import constants from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir @@ -23,7 +23,7 @@ def get_process_history(process_id): """ if not process_id: raise Exception("please pass `process_id` as a parameter to the function.") - r = requests.post(f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'process', 'id': process_id}).json() + r = requests.post(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'process', 'id': process_id}).json() return r @@ -34,5 +34,5 @@ def get_workflow_history(workflow_id): """ if not workflow_id: raise Exception("please pass `workflow_id` as a parameter to the function.") - r = requests.get(f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'workflow', 'id': workflow_id}).json() + r = requests.get(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'workflow', 'id': workflow_id}).json() return r From 9e52fe0977c7b0aee8fae8c2f887c8cee0edee13 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Wed, 7 Jun 2023 17:28:53 -0400 Subject: [PATCH 07/24] history commands use geoweaver cli if possible --- pygeoweaver/sc_history.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pygeoweaver/sc_history.py b/pygeoweaver/sc_history.py index 8e3b0e3..9f087a9 100644 --- a/pygeoweaver/sc_history.py +++ b/pygeoweaver/sc_history.py @@ -23,8 +23,14 @@ def get_process_history(process_id): """ if not process_id: raise Exception("please pass `process_id` as a parameter to the function.") - r = requests.post(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'process', 'id': process_id}).json() - return r + download_geoweaver_jar() + try: + subprocess.run(f"{get_java_bin_path()} -jar {get_geoweaver_jar_path()} process-history {process_id}", + cwd=f"{get_root_dir()}/", shell=True) + except subprocess.CalledProcessError as e: + r = requests.post(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", + data={'type': 'process', 'id': process_id}).json() + return r def get_workflow_history(workflow_id): @@ -34,5 +40,10 @@ def get_workflow_history(workflow_id): """ if not workflow_id: raise Exception("please pass `workflow_id` as a parameter to the function.") - r = requests.get(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'workflow', 'id': workflow_id}).json() - return r + try: + subprocess.run(f"{get_java_bin_path()} -jar {get_geoweaver_jar_path()} workflow-history {workflow_id}", + shell=True, cwd=f"{get_root_dir()}/") + except subprocess.CalledProcessError as e: + r = requests.get(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", + data={'type': 'workflow', 'id': workflow_id}).json() + return r From 5cb84e5639c1a7f9f889a3c8fc8d4a25a680547f Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 02:22:22 -0400 Subject: [PATCH 08/24] changes to create_process and create_workflow --- pygeoweaver/constants.py | 1 + pygeoweaver/sc_create.py | 100 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 pygeoweaver/sc_create.py diff --git a/pygeoweaver/constants.py b/pygeoweaver/constants.py index 2d08246..70bed80 100644 --- a/pygeoweaver/constants.py +++ b/pygeoweaver/constants.py @@ -1,2 +1,3 @@ GEOWEAVER_DEFAULT_ENDPOINT_URL="http://localhost:8070/Geoweaver" +COMMON_API_HEADER = {'Content-Type': 'application/json'} \ No newline at end of file diff --git a/pygeoweaver/sc_create.py b/pygeoweaver/sc_create.py new file mode 100644 index 0000000..b400e0d --- /dev/null +++ b/pygeoweaver/sc_create.py @@ -0,0 +1,100 @@ +import json +import requests +from pydantic import BaseModel + +from . import constants +from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir + + +class ProcessData(BaseModel): + type: str = "process" + lang: str + description: str + name: str + code: str + owner: str = "111111" + confidential: bool = False + + +class WorkflowData(BaseModel): + type: str = "workflow" + confidential: bool = False + description: str + edges: str + name: str + nodes: str + owner: str = "111111" + + +def create_process(lang, description, name, code, owner="111111", confidential=False): + """ + Function to create a process with given data if valid. + :param lang: The programming language of the process + :type lang: str + :param description: The description of the process + :type description: str + :param name: The name of the process + :type name: str + :param code: The code of the process + :type code: str + :param owner: The owner of the process, defaults to "111111" + :type owner: str, optional + :param confidential: The confidentiality status of the process, defaults to False + :type confidential: bool, optional + :return: Returns the id of the created process + :rtype: dict + """ + download_geoweaver_jar() + process = ProcessData( + type="process", + lang=lang, + description=description, + name=name, + code=code, + owner=owner, + confidential=confidential + ) + data_json = process.json() + r = requests.get( + f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/add/process", + data=data_json, + headers=constants.COMMON_API_HEADER + ) + return r.json() + + +def create_workflow(description, edges, name, nodes, owner="111111", confidential=False): + """ + Function to create a workflow with given data if valid + :param confidential: The confidentiality status of the workflow, defaults to False + :type confidential: bool, optional + :param description: The description of the workflow + :type description: str + :param edges: The edges of the workflow + :type edges: str + :param name: The name of the workflow + :type name: str + :param nodes: The nodes of the workflow + :type nodes: str + :param owner: The owner of the workflow, defaults to "111111" + :type owner: str, optional + :return: Returns the id of the created workflow + :rtype: dict + """ + download_geoweaver_jar() + workflow = WorkflowData( + type="workflow", + confidential=confidential, + description=description, + edges=edges, + name=name, + nodes=nodes, + owner=owner + ) + data_json = workflow.json() + r = requests.get( + f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/add/workflow", + data=data_json, + headers=constants.COMMON_API_HEADER + ) + return r.json() From b2063c0eabba9006f960c96b3a655bb8fffb3bca Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 02:25:49 -0400 Subject: [PATCH 09/24] add process, workflow creation functions to interface --- pygeoweaver/sc_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index fe11585..c49b165 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -11,3 +11,4 @@ from pygeoweaver.server import * from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * +from pygeoweaver.sc_create import * \ No newline at end of file From 47280c6aa0d6f3256d7733430283e9274bd6118d Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 02:30:14 -0400 Subject: [PATCH 10/24] changes to workflow, process --- pygeoweaver/sc_create.py | 4 ++-- pygeoweaver/sc_interface.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pygeoweaver/sc_create.py b/pygeoweaver/sc_create.py index b400e0d..3ec0e54 100644 --- a/pygeoweaver/sc_create.py +++ b/pygeoweaver/sc_create.py @@ -55,7 +55,7 @@ def create_process(lang, description, name, code, owner="111111", confidential=F confidential=confidential ) data_json = process.json() - r = requests.get( + r = requests.post( f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/add/process", data=data_json, headers=constants.COMMON_API_HEADER @@ -92,7 +92,7 @@ def create_workflow(description, edges, name, nodes, owner="111111", confidentia owner=owner ) data_json = workflow.json() - r = requests.get( + r = requests.post( f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/add/workflow", data=data_json, headers=constants.COMMON_API_HEADER diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index c49b165..c16988e 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -11,4 +11,4 @@ from pygeoweaver.server import * from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * -from pygeoweaver.sc_create import * \ No newline at end of file +from pygeoweaver.sc_create import * From 590b248b414db6161667df40d612c8d43b768a21 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 02:38:47 -0400 Subject: [PATCH 11/24] display process in a tabular format --- pygeoweaver/sc_create.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/pygeoweaver/sc_create.py b/pygeoweaver/sc_create.py index 3ec0e54..61d9c91 100644 --- a/pygeoweaver/sc_create.py +++ b/pygeoweaver/sc_create.py @@ -1,9 +1,12 @@ import json import requests +from IPython.core.display_functions import display +from ipywidgets import widgets from pydantic import BaseModel from . import constants -from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir +from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir, \ + check_ipython class ProcessData(BaseModel): @@ -60,7 +63,17 @@ def create_process(lang, description, name, code, owner="111111", confidential=F data=data_json, headers=constants.COMMON_API_HEADER ) - return r.json() + if check_ipython() and r.ok: + data = json.loads(data_json) + header_labels = ['Key', 'Value'] + header_widgets = [widgets.Label(value=label) for label in header_labels] + row_widgets = [widgets.Label(value=str(item)) for item in data.items()] + grid = widgets.GridBox(header_widgets + row_widgets, + layout=widgets.Layout(grid_template_columns="repeat(2, auto)")) + grid.layout.border = '1px solid black' + display(grid) + else: + return r.json() def create_workflow(description, edges, name, nodes, owner="111111", confidential=False): @@ -97,4 +110,7 @@ def create_workflow(description, edges, name, nodes, owner="111111", confidentia data=data_json, headers=constants.COMMON_API_HEADER ) - return r.json() + if check_ipython(): + pass + else: + return r.json() From 905cdc815bb6853682ebd8f11fa02316b54d9439 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 02:48:53 -0400 Subject: [PATCH 12/24] show dataframe table instead of normal text --- pygeoweaver/sc_create.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pygeoweaver/sc_create.py b/pygeoweaver/sc_create.py index 61d9c91..2597566 100644 --- a/pygeoweaver/sc_create.py +++ b/pygeoweaver/sc_create.py @@ -1,7 +1,6 @@ import json import requests -from IPython.core.display_functions import display -from ipywidgets import widgets +import pandas as pd from pydantic import BaseModel from . import constants @@ -64,14 +63,8 @@ def create_process(lang, description, name, code, owner="111111", confidential=F headers=constants.COMMON_API_HEADER ) if check_ipython() and r.ok: - data = json.loads(data_json) - header_labels = ['Key', 'Value'] - header_widgets = [widgets.Label(value=label) for label in header_labels] - row_widgets = [widgets.Label(value=str(item)) for item in data.items()] - grid = widgets.GridBox(header_widgets + row_widgets, - layout=widgets.Layout(grid_template_columns="repeat(2, auto)")) - grid.layout.border = '1px solid black' - display(grid) + df = pd.DataFrame(json.loads(data_json).items(), columns=['Key', 'Value']) + return df else: return r.json() @@ -111,6 +104,6 @@ def create_workflow(description, edges, name, nodes, owner="111111", confidentia headers=constants.COMMON_API_HEADER ) if check_ipython(): - pass + return pd.DataFrame(json.loads(data_json).items(), columns=['Key', 'Value']) else: return r.json() From 530cc133656b37c0b1408792d3dd27eda5e169a5 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 03:47:03 -0400 Subject: [PATCH 13/24] add pydantic to project deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8ca3ce2..33f9adf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ homepage = "https://github.com/ESIPFed/pygeoweaver" python = ">=3.7" setuptools = ">=61.0" requests = "2.28.2" +pydantic = "1.10.9" [tool.poetry.scripts] From c7a4baf63edaa34d845941113fbd98fae2e2562e Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 03:57:22 -0400 Subject: [PATCH 14/24] added function to read file from path --- pygeoweaver/sc_create.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pygeoweaver/sc_create.py b/pygeoweaver/sc_create.py index 2597566..f884e2e 100644 --- a/pygeoweaver/sc_create.py +++ b/pygeoweaver/sc_create.py @@ -69,6 +69,29 @@ def create_process(lang, description, name, code, owner="111111", confidential=F return r.json() +def create_process_from_file(lang, description, name, file_path, owner="111111", confidential=False): + """ + Function to create a process with code from a file. + :param lang: The programming language of the process. + :type lang: str + :param description: The description of the process. + :type description: str + :param name: The name of the process. + :type name: str + :param file_path: The path to the file containing the code. + :type file_path: str + :param owner: The owner of the process, defaults to "111111". + :type owner: str, optional + :param confidential: The confidentiality status of the process, defaults to False. + :type confidential: bool, optional + :return: Returns the id of the created process. + :rtype: dict + """ + with open(file_path, 'r') as file: + code = file.read() + return create_process(lang, description, name, code, owner=owner, confidential=confidential) + + def create_workflow(description, edges, name, nodes, owner="111111", confidential=False): """ Function to create a workflow with given data if valid From df1bfc1c2a5b830e471750c1493fd0234ad4c0e6 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 13:52:00 -0400 Subject: [PATCH 15/24] try api first then if fails try cli --- pygeoweaver/sc_history.py | 18 ++++++++++-------- pygeoweaver/sc_sync.py | 7 +++++++ 2 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 pygeoweaver/sc_sync.py diff --git a/pygeoweaver/sc_history.py b/pygeoweaver/sc_history.py index 9f087a9..3fdb348 100644 --- a/pygeoweaver/sc_history.py +++ b/pygeoweaver/sc_history.py @@ -1,4 +1,6 @@ import subprocess + +import pandas as pd import requests from . import constants @@ -25,12 +27,12 @@ def get_process_history(process_id): raise Exception("please pass `process_id` as a parameter to the function.") download_geoweaver_jar() try: - subprocess.run(f"{get_java_bin_path()} -jar {get_geoweaver_jar_path()} process-history {process_id}", - cwd=f"{get_root_dir()}/", shell=True) - except subprocess.CalledProcessError as e: r = requests.post(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'process', 'id': process_id}).json() - return r + return pd.DataFrame(r) + except Exception as e: + subprocess.run(f"{get_java_bin_path()} -jar {get_geoweaver_jar_path()} process-history {process_id}", + cwd=f"{get_root_dir()}/", shell=True) def get_workflow_history(workflow_id): @@ -41,9 +43,9 @@ def get_workflow_history(workflow_id): if not workflow_id: raise Exception("please pass `workflow_id` as a parameter to the function.") try: - subprocess.run(f"{get_java_bin_path()} -jar {get_geoweaver_jar_path()} workflow-history {workflow_id}", - shell=True, cwd=f"{get_root_dir()}/") - except subprocess.CalledProcessError as e: r = requests.get(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'workflow', 'id': workflow_id}).json() - return r + return pd.DataFrame(r) + except Exception as e: + subprocess.run(f"{get_java_bin_path()} -jar {get_geoweaver_jar_path()} workflow-history {workflow_id}", + shell=True, cwd=f"{get_root_dir()}/") diff --git a/pygeoweaver/sc_sync.py b/pygeoweaver/sc_sync.py new file mode 100644 index 0000000..b770e2f --- /dev/null +++ b/pygeoweaver/sc_sync.py @@ -0,0 +1,7 @@ +import subprocess +import requests + +from . import constants +from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir + + From 901acca55e83d006ee28351089d365e9f8356bbe Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 14:07:16 -0400 Subject: [PATCH 16/24] changes to workflow api --- pygeoweaver/sc_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeoweaver/sc_history.py b/pygeoweaver/sc_history.py index 3fdb348..60d4b6a 100644 --- a/pygeoweaver/sc_history.py +++ b/pygeoweaver/sc_history.py @@ -43,7 +43,7 @@ def get_workflow_history(workflow_id): if not workflow_id: raise Exception("please pass `workflow_id` as a parameter to the function.") try: - r = requests.get(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", + r = requests.post(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", data={'type': 'workflow', 'id': workflow_id}).json() return pd.DataFrame(r) except Exception as e: From 20030eacacc3dea63bcc5414519dad44293069f4 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 15:58:02 -0400 Subject: [PATCH 17/24] sync files on demand --- pygeoweaver/sc_history.py | 2 +- pygeoweaver/sc_sync.py | 35 +++++++++++++++++++++++++++++++++-- pygeoweaver/utils.py | 35 ++++++++++++++++++++--------------- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/pygeoweaver/sc_history.py b/pygeoweaver/sc_history.py index 60d4b6a..12e7449 100644 --- a/pygeoweaver/sc_history.py +++ b/pygeoweaver/sc_history.py @@ -44,7 +44,7 @@ def get_workflow_history(workflow_id): raise Exception("please pass `workflow_id` as a parameter to the function.") try: r = requests.post(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/logs", - data={'type': 'workflow', 'id': workflow_id}).json() + data={'type': 'workflow', 'id': workflow_id}).json() return pd.DataFrame(r) except Exception as e: subprocess.run(f"{get_java_bin_path()} -jar {get_geoweaver_jar_path()} workflow-history {workflow_id}", diff --git a/pygeoweaver/sc_sync.py b/pygeoweaver/sc_sync.py index b770e2f..154248c 100644 --- a/pygeoweaver/sc_sync.py +++ b/pygeoweaver/sc_sync.py @@ -1,7 +1,38 @@ -import subprocess +import os +import json +import zipfile + import requests +import typing from . import constants -from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir +from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir, \ + copy_files + + +def sync_workflow(workflow_id: str, sync_to_path: typing.Union[str, os.PathLike]): + # download workflow + r = requests.post(f'{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/downloadworkflow', + data={'id': workflow_id, 'option': 'workflowwithprocesscodeallhistory'}, + headers=constants.COMMON_API_HEADER).text + filename = r.rsplit('/')[-1] + home_dir = os.path.expanduser("~") + tmp_dir = os.path.join(home_dir, 'tmp') + if not os.path.exists(tmp_dir): + os.makedirs(tmp_dir) + # unzip the workflow + with zipfile.ZipFile(os.path.join(home_dir, 'gw-workspace', 'temp', filename)) as ref: + ref.extractall(os.path.join(home_dir, 'tmp')) + # check if target workflow path and the unzipped workflow match + if not sync_to_path: + raise Exception("Please provide path to workflow that you wish to sync code and history") + import_id = json.loads(open(os.path.join(home_dir, 'tmp', 'workflow.json'), "r").read()).get("id") + sync_id = json.loads(open(sync_to_path, "r").read()).get("id") + + if import_id == sync_id: + # if they match perform file replace + copy_files(source_folder=os.path.join(home_dir, 'tmp'), destination_folder=sync_to_path) + else: + print("Workflow ID mismatch, please check the `sync_to_path` path.") diff --git a/pygeoweaver/utils.py b/pygeoweaver/utils.py index 58f2d4a..c026bf9 100644 --- a/pygeoweaver/utils.py +++ b/pygeoweaver/utils.py @@ -1,14 +1,12 @@ -import logging import os +import sys +import shutil +import logging import subprocess import requests import platform -import sys from IPython import get_ipython -import os -import sys - def get_home_dir(): @@ -21,11 +19,10 @@ def get_root_dir(): def get_java_bin_from_which(): - system = platform.system() if system == 'Darwin' or system == 'Linux': - + try: java_bin_sh = f'{get_root_dir()}/java_bin.sh' @@ -33,13 +30,13 @@ def get_java_bin_from_which(): os.chmod(java_bin_sh, 0o755) output = subprocess.check_output([java_bin_sh], encoding='utf-8') - + java_bin_path = output.strip() except subprocess.CalledProcessError as e: print(f"Command execution failed: {e.output}") - + return None elif system == 'Windows': @@ -48,9 +45,8 @@ def get_java_bin_from_which(): else: print('Unsupported platform.') - - return java_bin_path + return java_bin_path def get_java_bin_path(): @@ -59,18 +55,18 @@ def get_java_bin_path(): java_exe = 'java.exe' else: java_exe = 'java' - + java_bin_path = None - + for path in os.environ.get('PATH', '').split(os.pathsep): bin_path = os.path.join(path, java_exe) if os.path.isfile(bin_path) and os.access(bin_path, os.X_OK): java_bin_path = bin_path break - + if java_bin_path is None: java_bin_path = get_java_bin_from_which() - + return java_bin_path @@ -135,3 +131,12 @@ def get_logger(class_name): console_handler.setFormatter(formatter) logger.addHandler(console_handler) return logger + + +def copy_files(source_folder, destination_folder): + for root, dirs, files in os.walk(source_folder): + for file in files: + source_file = os.path.join(root, file) + destination_file = os.path.join(destination_folder, os.path.relpath(source_file, source_folder)) + os.makedirs(os.path.dirname(destination_file), exist_ok=True) + shutil.copy2(source_file, destination_file) From fee4187dd969346ba2935428cff2268296bf8c92 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 15:58:57 -0400 Subject: [PATCH 18/24] cleanup --- pygeoweaver/sc_sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygeoweaver/sc_sync.py b/pygeoweaver/sc_sync.py index 154248c..b1fb4f3 100644 --- a/pygeoweaver/sc_sync.py +++ b/pygeoweaver/sc_sync.py @@ -11,6 +11,7 @@ def sync_workflow(workflow_id: str, sync_to_path: typing.Union[str, os.PathLike]): + download_geoweaver_jar() # download workflow r = requests.post(f'{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/downloadworkflow', data={'id': workflow_id, 'option': 'workflowwithprocesscodeallhistory'}, From c08a5fa1a2757996d222da61e8bb2bd612231a5e Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 16:27:54 -0400 Subject: [PATCH 19/24] fixes to sync --- pygeoweaver/sc_sync.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pygeoweaver/sc_sync.py b/pygeoweaver/sc_sync.py index b1fb4f3..4da4321 100644 --- a/pygeoweaver/sc_sync.py +++ b/pygeoweaver/sc_sync.py @@ -14,8 +14,7 @@ def sync_workflow(workflow_id: str, sync_to_path: typing.Union[str, os.PathLike] download_geoweaver_jar() # download workflow r = requests.post(f'{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/downloadworkflow', - data={'id': workflow_id, 'option': 'workflowwithprocesscodeallhistory'}, - headers=constants.COMMON_API_HEADER).text + data={'id': workflow_id, 'option': 'workflowwithprocesscodeallhistory'}).text filename = r.rsplit('/')[-1] home_dir = os.path.expanduser("~") tmp_dir = os.path.join(home_dir, 'tmp') @@ -28,7 +27,7 @@ def sync_workflow(workflow_id: str, sync_to_path: typing.Union[str, os.PathLike] if not sync_to_path: raise Exception("Please provide path to workflow that you wish to sync code and history") import_id = json.loads(open(os.path.join(home_dir, 'tmp', 'workflow.json'), "r").read()).get("id") - sync_id = json.loads(open(sync_to_path, "r").read()).get("id") + sync_id = json.loads(open(os.path.join(sync_to_path, "workflow.json"), "r").read()).get("id") if import_id == sync_id: # if they match perform file replace From 9dd46fb45cab14b3142c7eaecbd0e01a4ada644f Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 16:29:00 -0400 Subject: [PATCH 20/24] added sync to interface --- pygeoweaver/sc_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index c16988e..0922925 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -12,3 +12,4 @@ from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * from pygeoweaver.sc_create import * +from pygeoweaver.sc_sync import * \ No newline at end of file From 7f5b83643dcffae38b334104a975b1e2f4ce5c03 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 17:50:01 -0400 Subject: [PATCH 21/24] run process with sync --- pygeoweaver/sc_interface.py | 2 +- pygeoweaver/sc_run.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index 0922925..07bdc4b 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -12,4 +12,4 @@ from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * from pygeoweaver.sc_create import * -from pygeoweaver.sc_sync import * \ No newline at end of file +from pygeoweaver.sc_sync import * diff --git a/pygeoweaver/sc_run.py b/pygeoweaver/sc_run.py index a992dd0..bf794c6 100644 --- a/pygeoweaver/sc_run.py +++ b/pygeoweaver/sc_run.py @@ -1,5 +1,10 @@ +import json import os import subprocess + +import requests + +from . import constants from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir @@ -16,7 +21,8 @@ def create_workflow(): pass -def run_process(*, process_id: str, host_id: str, password: str, environment: str = None): +def run_process(*, process_id: str, host_id: str, password: str, environment: str = None, + sync_path: os.PathLike = None): """ Run a process @@ -25,6 +31,35 @@ def run_process(*, process_id: str, host_id: str, password: str, environment: st password - required environment - optional """ + if sync_path: + ext, matching_dict = None, None + process_file = os.path.exists(os.path.join(sync_path, 'code', 'process.json')) + if not process_file: + print("process file does not exists, please check the path") + return + p_file = json.loads(open(os.path.join(sync_path, 'code', 'process.json'), "r").read()) + for item in p_file: + if item.get("id") == process_id: + matching_dict = item + break + if not matching_dict: + print("Could not find the file, please check the path") + return + if matching_dict['lang'] == "python": + ext = ".py" + if matching_dict['lang'] == "bash": + ext = ".bash" + if not ext: + print("Invalid file format.") + source_filename = matching_dict['name'] + ext + source_file_exists = os.path.exists(os.path.join(sync_path, 'code', source_filename)) + if source_file_exists: + f = open(os.path.join(sync_path, 'code', source_filename), "r").read() + matching_dict['code'] = f + requests.post(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/edit/process", + data=json.dumps(matching_dict), headers={'Content-Type': 'application/json'}) + else: + print("File does not exists") download_geoweaver_jar() subprocess.run([get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "run", "process", f"--host={host_id}", f"--password={password}", f"--environment={environment}", process_id], From ec0bb41e5272d756a2ee534331b836936f73c047 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 20:30:21 -0400 Subject: [PATCH 22/24] added workflow sync --- pygeoweaver/sc_run.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pygeoweaver/sc_run.py b/pygeoweaver/sc_run.py index bf794c6..1785bb4 100644 --- a/pygeoweaver/sc_run.py +++ b/pygeoweaver/sc_run.py @@ -4,7 +4,7 @@ import requests -from . import constants +from . import constants, sync_workflow from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir @@ -57,7 +57,7 @@ def run_process(*, process_id: str, host_id: str, password: str, environment: st f = open(os.path.join(sync_path, 'code', source_filename), "r").read() matching_dict['code'] = f requests.post(f"{constants.GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/edit/process", - data=json.dumps(matching_dict), headers={'Content-Type': 'application/json'}) + data=json.dumps(matching_dict), headers={'Content-Type': 'application/json'}) else: print("File does not exists") download_geoweaver_jar() @@ -67,7 +67,8 @@ def run_process(*, process_id: str, host_id: str, password: str, environment: st def run_workflow(*, workflow_id: str, workflow_folder_path: str = None, workflow_zip_file_path: str = None, - environment_list: str = None, host_list: str = None, password_list: str = None): + environment_list: str = None, host_list: str = None, password_list: str = None, + sync_path: os.PathLike = None): """ Usage:
run workflow [-d=] [-f=] [-e=]... @@ -84,6 +85,9 @@ def run_workflow(*, workflow_id: str, workflow_folder_path: str = None, workflow """ download_geoweaver_jar() + if sync_path: + sync_workflow(workflow_id=workflow_id, sync_to_path=sync_path) + if not workflow_id and not workflow_folder_path and not workflow_zip_file_path: raise RuntimeError("Please provide at least one of the three options: workflow id, " "folder path or zip path") @@ -109,4 +113,3 @@ def run_workflow(*, workflow_id: str, workflow_folder_path: str = None, workflow command.extend(["-e", environment_list]) command.extend(["-f", workflow_zip_file_path, "-h", host_list, "-p", password_list]) subprocess.run(command, cwd=f"{get_root_dir()}/") - From 2d4e07db461ad0ed0d5f0110ef2d8114d06b3a0b Mon Sep 17 00:00:00 2001 From: Ziheng Sun Date: Fri, 9 Jun 2023 21:06:00 -0400 Subject: [PATCH 23/24] update --- pygeoweaver/jdk_utils.py | 8 ++++---- pygeoweaver/start.sh | 2 +- pyproject.toml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pygeoweaver/jdk_utils.py b/pygeoweaver/jdk_utils.py index a601fd7..7be2b2c 100644 --- a/pygeoweaver/jdk_utils.py +++ b/pygeoweaver/jdk_utils.py @@ -62,10 +62,10 @@ def install_jdk_linux(jdk_version, jdk_arch): jdk_install_dir = os.path.expanduser('~/jdk') # Download JDK archive - download_file(jdk_url, 'jdk.tar.gz') + download_file(jdk_url, f'{get_home_dir()}/jdk.tar.gz') # Extract JDK archive - extract_tar_archive('jdk.tar.gz', jdk_install_dir) + extract_tar_archive(f'{get_home_dir()}/jdk.tar.gz', jdk_install_dir) # Set JDK environment variables set_jdk_env_vars(f'{jdk_install_dir}/jdk-{jdk_version.replace("-", "+")}') @@ -76,10 +76,10 @@ def install_jdk_windows(jdk_version, jdk_arch): jdk_install_dir = os.path.expanduser('~/jdk') # Download JDK archive - download_file(jdk_url, 'jdk.zip') + download_file(jdk_url, f'{get_home_dir()}/jdk.zip') # Extract JDK archive - extract_zip_archive('jdk.zip', jdk_install_dir) + extract_zip_archive(f'{get_home_dir()}/jdk.zip', jdk_install_dir) # Set JDK environment variables set_jdk_env_vars(f'{jdk_install_dir}/jdk-{jdk_version.replace("-", "+")}') diff --git a/pygeoweaver/start.sh b/pygeoweaver/start.sh index d9f39c8..3b5e316 100755 --- a/pygeoweaver/start.sh +++ b/pygeoweaver/start.sh @@ -4,7 +4,7 @@ echo "Stop running Geoweaver if any.." pkill -f geoweaver.jar echo "Check Java.." -source ~/.bashrc +touch ~/.bashrc && source ~/.bashrc echo "Start Geoweaver.." nohup java -jar ~/geoweaver.jar > ~/geoweaver.log & diff --git a/pyproject.toml b/pyproject.toml index 8ca3ce2..de9dec9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pygeoweaver" -version = "0.6.14" +version = "0.6.15" authors = [ { name="Geoweaver team", email="geoweaver.app@gmail.com" }, ] @@ -22,7 +22,7 @@ classifiers = [ [tool.poetry] name = "pygeoweaver" -version = "0.6.14" +version = "0.6.15" description = "This is a wrapper package of the Geoweaver app." authors = ["Geoweaver team "] readme = "README.md" From aeb5cc52090be6076871f232120a96cb943f830c Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Fri, 9 Jun 2023 22:05:45 -0400 Subject: [PATCH 24/24] added workflow sync --- pygeoweaver/sc_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pygeoweaver/sc_run.py b/pygeoweaver/sc_run.py index 1785bb4..c614597 100644 --- a/pygeoweaver/sc_run.py +++ b/pygeoweaver/sc_run.py @@ -4,7 +4,7 @@ import requests -from . import constants, sync_workflow +from . import constants from pygeoweaver.utils import download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir @@ -86,6 +86,7 @@ def run_workflow(*, workflow_id: str, workflow_folder_path: str = None, workflow download_geoweaver_jar() if sync_path: + from . import sync_workflow sync_workflow(workflow_id=workflow_id, sync_to_path=sync_path) if not workflow_id and not workflow_folder_path and not workflow_zip_file_path: