From 5237ba9e4770afedb4542a30c424c81c6a4ace14 Mon Sep 17 00:00:00 2001 From: ligerlac Date: Thu, 11 Apr 2024 22:09:17 +0200 Subject: [PATCH 1/7] context manager for env variables --- common/lib/idds/common/utils.py | 32 +++++++++++++++++++++++++ workflow/lib/idds/iworkflow/work.py | 7 +++--- workflow/lib/idds/iworkflow/workflow.py | 7 +++--- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/common/lib/idds/common/utils.py b/common/lib/idds/common/utils.py index 1a73cde2..f2470343 100644 --- a/common/lib/idds/common/utils.py +++ b/common/lib/idds/common/utils.py @@ -25,6 +25,7 @@ import tarfile import time # import traceback +import contextlib from enum import Enum from functools import wraps @@ -942,3 +943,34 @@ def idds_mask(dict_): else: ret[k] = dict_[k] return ret + + +@contextlib.contextmanager +def modified_environ(*remove, **update): + """ + Temporarily updates the ``os.environ`` dictionary in-place. + + The ``os.environ`` dictionary is updated in-place so that the modification + is sure to work in all situations. + + :param remove: Environment variables to remove. + :param update: Dictionary of environment variables and values to add/update. + """ + env = os.environ + update = update or {} + remove = remove or [] + + # List of environment variables being updated or removed. + stomped = (set(update.keys()) | set(remove)) & set(env.keys()) + # Environment variables and values to restore on exit. + update_after = {k: env[k] for k in stomped} + # Environment variables and values to remove on exit. + remove_after = frozenset(k for k in update if k not in env) + + try: + env.update(update) + [env.pop(k, None) for k in remove] + yield + finally: + env.update(update_after) + [env.pop(k) for k in remove_after] \ No newline at end of file diff --git a/workflow/lib/idds/iworkflow/work.py b/workflow/lib/idds/iworkflow/work.py index f15e9cb3..18f57138 100644 --- a/workflow/lib/idds/iworkflow/work.py +++ b/workflow/lib/idds/iworkflow/work.py @@ -779,10 +779,9 @@ def load(self, func_name): :raise Exception """ - os.environ['IDDS_IWORKFLOW_LOAD_WORK'] = 'true' - func = super(Work, self).load(func_name) - del os.environ['IDDS_IWORKFLOW_LOAD_WORK'] - + import idds.common.utils as idds_utils + with idds_utils.modified_environ(IDDS_IWORKFLOW_LOAD_WORK='true'): + func = super(Work, self).load(func_name) return func def pre_run(self): diff --git a/workflow/lib/idds/iworkflow/workflow.py b/workflow/lib/idds/iworkflow/workflow.py index cf4eb719..baac27df 100644 --- a/workflow/lib/idds/iworkflow/workflow.py +++ b/workflow/lib/idds/iworkflow/workflow.py @@ -1059,10 +1059,9 @@ def load(self, func_name): :raise Exception """ - os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] = 'true' - func = super(Workflow, self).load(func_name) - del os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] - + import idds.common.utils as idds_utils + with idds_utils.modified_environ(IDDS_IWORKFLOW_LOAD_WORKFLOW='true'): + func = super(Workflow, self).load(func_name) return func def pre_run(self): From d4ac132d437426e0d031fbadabe8eae931862624 Mon Sep 17 00:00:00 2001 From: ligerlac Date: Thu, 11 Apr 2024 22:54:41 +0200 Subject: [PATCH 2/7] remove redundant logic --- workflow/lib/idds/iworkflow/workflow.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/workflow/lib/idds/iworkflow/workflow.py b/workflow/lib/idds/iworkflow/workflow.py index baac27df..41a6c79e 100644 --- a/workflow/lib/idds/iworkflow/workflow.py +++ b/workflow/lib/idds/iworkflow/workflow.py @@ -638,9 +638,7 @@ def get_idds_env(self): def get_idds_server(self): if 'IDDS_HOST' in self._idds_env: return self._idds_env['IDDS_HOST'] - if os.environ.get('IDDS_HOST', None): - return os.environ.get('IDDS_HOST', None) - return None + return os.environ.get('IDDS_HOST', None) def prepare_with_idds(self): """ From c01e19f1e5dfd9236764e8e33f4e374f57e60b5c Mon Sep 17 00:00:00 2001 From: ligerlac Date: Fri, 12 Apr 2024 00:00:01 +0200 Subject: [PATCH 3/7] rename env var for more clarity --- workflow/lib/idds/iworkflow/work.py | 4 ++-- workflow/lib/idds/iworkflow/workflow.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/lib/idds/iworkflow/work.py b/workflow/lib/idds/iworkflow/work.py index 18f57138..4b134535 100644 --- a/workflow/lib/idds/iworkflow/work.py +++ b/workflow/lib/idds/iworkflow/work.py @@ -780,7 +780,7 @@ def load(self, func_name): :raise Exception """ import idds.common.utils as idds_utils - with idds_utils.modified_environ(IDDS_IWORKFLOW_LOAD_WORK='true'): + with idds_utils.modified_environ(IDDS_IGNORE_WORK_DECORATOR='true'): func = super(Work, self).load(func_name) return func @@ -921,7 +921,7 @@ def work(func=None, *, map_results=False, lazy=False, init_env=None): if func is None: return functools.partial(work, map_results=map_results, lazy=lazy, init_env=init_env) - if 'IDDS_IWORKFLOW_LOAD_WORK' in os.environ: + if 'IDDS_IGNORE_WORK_DECORATOR' in os.environ: return func @functools.wraps(func) diff --git a/workflow/lib/idds/iworkflow/workflow.py b/workflow/lib/idds/iworkflow/workflow.py index 41a6c79e..be0ec25a 100644 --- a/workflow/lib/idds/iworkflow/workflow.py +++ b/workflow/lib/idds/iworkflow/workflow.py @@ -1058,7 +1058,7 @@ def load(self, func_name): :raise Exception """ import idds.common.utils as idds_utils - with idds_utils.modified_environ(IDDS_IWORKFLOW_LOAD_WORKFLOW='true'): + with idds_utils.modified_environ(IDDS_IGNORE_WORKFLOW_DECORATOR='true'): func = super(Workflow, self).load(func_name) return func @@ -1158,7 +1158,7 @@ def workflow(func=None, *, local=False, service='idds', source_dir=None, primary return functools.partial(workflow, local=local, service=service, source_dir=source_dir, primary=primary, queue=queue, site=site, cloud=cloud, max_walltime=max_walltime, distributed=distributed, init_env=init_env) - if 'IDDS_IWORKFLOW_LOAD_WORKFLOW' in os.environ: + if 'IDDS_IGNORE_WORKFLOW_DECORATOR' in os.environ: return func @functools.wraps(func) From d58381662df570b379ddde4319ff4291493f8096 Mon Sep 17 00:00:00 2001 From: ligerlac Date: Fri, 12 Apr 2024 16:10:52 +0200 Subject: [PATCH 4/7] move import statements to the top --- workflow/lib/idds/iworkflow/work.py | 5 ++--- workflow/lib/idds/iworkflow/workflow.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/workflow/lib/idds/iworkflow/work.py b/workflow/lib/idds/iworkflow/work.py index 4b134535..41663271 100644 --- a/workflow/lib/idds/iworkflow/work.py +++ b/workflow/lib/idds/iworkflow/work.py @@ -21,7 +21,7 @@ from idds.common import exceptions from idds.common.constants import WorkflowType, TransformStatus from idds.common.imports import get_func_name -from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64 +from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64, modified_environ from .asyncresult import AsyncResult, MapResult from .base import Base, Context from .workflow import WorkflowCanvas @@ -779,8 +779,7 @@ def load(self, func_name): :raise Exception """ - import idds.common.utils as idds_utils - with idds_utils.modified_environ(IDDS_IGNORE_WORK_DECORATOR='true'): + with modified_environ(IDDS_IGNORE_WORK_DECORATOR='true'): func = super(Work, self).load(func_name) return func diff --git a/workflow/lib/idds/iworkflow/workflow.py b/workflow/lib/idds/iworkflow/workflow.py index be0ec25a..5d1ee912 100644 --- a/workflow/lib/idds/iworkflow/workflow.py +++ b/workflow/lib/idds/iworkflow/workflow.py @@ -23,7 +23,8 @@ # from idds.common import exceptions from idds.common.constants import WorkflowType -from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64 +from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64,\ + modified_environ from .asyncresult import AsyncResult from .base import Base, Context @@ -1057,8 +1058,7 @@ def load(self, func_name): :raise Exception """ - import idds.common.utils as idds_utils - with idds_utils.modified_environ(IDDS_IGNORE_WORKFLOW_DECORATOR='true'): + with modified_environ(IDDS_IGNORE_WORKFLOW_DECORATOR='true'): func = super(Workflow, self).load(func_name) return func From d5293bd9a92a3d8a9e805b577214845b9eea9440 Mon Sep 17 00:00:00 2001 From: wguanicedew Date: Fri, 12 Apr 2024 16:52:20 +0200 Subject: [PATCH 5/7] Revert "Env var to ignore work / workflow decorators" --- common/lib/idds/common/utils.py | 32 ------------------------- main/etc/sql/postgres_partition.sql | 15 ------------ main/etc/sql/postgresql.sql | 15 ++++++------ website/data/index.html | 2 +- workflow/lib/idds/iworkflow/work.py | 10 ++++---- workflow/lib/idds/iworkflow/workflow.py | 15 +++++++----- 6 files changed, 24 insertions(+), 65 deletions(-) delete mode 100644 main/etc/sql/postgres_partition.sql diff --git a/common/lib/idds/common/utils.py b/common/lib/idds/common/utils.py index f2470343..1a73cde2 100644 --- a/common/lib/idds/common/utils.py +++ b/common/lib/idds/common/utils.py @@ -25,7 +25,6 @@ import tarfile import time # import traceback -import contextlib from enum import Enum from functools import wraps @@ -943,34 +942,3 @@ def idds_mask(dict_): else: ret[k] = dict_[k] return ret - - -@contextlib.contextmanager -def modified_environ(*remove, **update): - """ - Temporarily updates the ``os.environ`` dictionary in-place. - - The ``os.environ`` dictionary is updated in-place so that the modification - is sure to work in all situations. - - :param remove: Environment variables to remove. - :param update: Dictionary of environment variables and values to add/update. - """ - env = os.environ - update = update or {} - remove = remove or [] - - # List of environment variables being updated or removed. - stomped = (set(update.keys()) | set(remove)) & set(env.keys()) - # Environment variables and values to restore on exit. - update_after = {k: env[k] for k in stomped} - # Environment variables and values to remove on exit. - remove_after = frozenset(k for k in update if k not in env) - - try: - env.update(update) - [env.pop(k, None) for k in remove] - yield - finally: - env.update(update_after) - [env.pop(k) for k in remove_after] \ No newline at end of file diff --git a/main/etc/sql/postgres_partition.sql b/main/etc/sql/postgres_partition.sql deleted file mode 100644 index 42eb4843..00000000 --- a/main/etc/sql/postgres_partition.sql +++ /dev/null @@ -1,15 +0,0 @@ -SELECT partman.create_parent( -p_parent_table => 'doma_idds.contents', -p_control => 'request_id', -p_type => 'native', -p_interval=> '1000', -p_premake => 3 -); - -SELECT partman.create_parent( -p_parent_table => 'doma_idds.contents_ext', -p_control => 'request_id', -p_type => 'native', -p_interval=> '1000', -p_premake => 3 -); \ No newline at end of file diff --git a/main/etc/sql/postgresql.sql b/main/etc/sql/postgresql.sql index c8d1f1d1..792a94cf 100644 --- a/main/etc/sql/postgresql.sql +++ b/main/etc/sql/postgresql.sql @@ -181,8 +181,8 @@ CREATE TABLE doma_idds.contents_ext ( memory_leak VARCHAR(10), memory_leak_x2 VARCHAR(10), job_label VARCHAR(20), - CONSTRAINT "CONTENTS_EXT_PK" PRIMARY KEY (content_id, request_id) -) PARTITION BY RANGE (request_id) ; + CONSTRAINT "CONTENTS_EXT_PK" PRIMARY KEY (content_id) +); CREATE INDEX "CONTENTS_EXT_RTW_IDX" ON doma_idds.contents_ext (request_id, transform_id, workload_id); @@ -500,13 +500,13 @@ CREATE TABLE doma_idds.contents ( accessed_at TIMESTAMP WITHOUT TIME ZONE, expired_at TIMESTAMP WITHOUT TIME ZONE, content_metadata VARCHAR(1000), - CONSTRAINT "CONTENTS_PK_TEST" PRIMARY KEY (content_id, request_id), - CONSTRAINT "CONTENT_ID_UQ_TEST" UNIQUE (transform_id, coll_id, request_id, map_id, sub_map_id, dep_sub_map_id, content_relation_type, name_md5, scope_name_md5, min_id, max_id), - CONSTRAINT "CONTENTS_TRANSFORM_ID_FK" FOREIGN KEY(transform_id) REFERENCES doma_idds.transforms (transform_id), + CONSTRAINT "CONTENTS_PK" PRIMARY KEY (content_id), + CONSTRAINT "CONTENT_ID_UQ" UNIQUE (transform_id, coll_id, map_id, sub_map_id, dep_sub_map_id, content_relation_type, name_md5, scope_name_md5, min_id, max_id), + CONSTRAINT "CONTENTS_TRANSFORM_ID_FK" FOREIGN KEY(transform_id) REFERENCES doma_idds.transforms (transform_id), CONSTRAINT "CONTENTS_COLL_ID_FK" FOREIGN KEY(coll_id) REFERENCES doma_idds.collections (coll_id), CONSTRAINT "CONTENTS_STATUS_ID_NN" CHECK (status IS NOT NULL), CONSTRAINT "CONTENTS_COLL_ID_NN" CHECK (coll_id IS NOT NULL) -) PARTITION BY RANGE (request_id) ; +); CREATE INDEX "CONTENTS_STATUS_UPDATED_IDX" ON doma_idds.contents (status, locking, updated_at, created_at); @@ -518,7 +518,8 @@ CREATE INDEX "CONTENTS_REQ_TF_COLL_IDX" ON doma_idds.contents (request_id, trans CREATE INDEX "CONTENTS_TF_IDX" ON doma_idds.contents (transform_id, request_id, coll_id, map_id, content_relation_type); -CREATE INDEX "CONTENTS_ID_NAME_IDX" ON doma_idds.contents (coll_id, scope, md5('name'), status); +CREATE INDEX "CONTENTS_ID_NAME_IDX" ON doma_idds.contents (coll_id, scope, md5('name'); +, status); SET search_path TO doma_idds; diff --git a/website/data/index.html b/website/data/index.html index 01434d84..5535ef1f 100644 --- a/website/data/index.html +++ b/website/data/index.html @@ -61,7 +61,7 @@

Intelligent

Fine grained data transformation

-

Remote data transformation/reduction

+

Remtoe data transformation/reduction

On-demand production of analysis format data transformation

Transformation

diff --git a/workflow/lib/idds/iworkflow/work.py b/workflow/lib/idds/iworkflow/work.py index 41663271..f15e9cb3 100644 --- a/workflow/lib/idds/iworkflow/work.py +++ b/workflow/lib/idds/iworkflow/work.py @@ -21,7 +21,7 @@ from idds.common import exceptions from idds.common.constants import WorkflowType, TransformStatus from idds.common.imports import get_func_name -from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64, modified_environ +from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64 from .asyncresult import AsyncResult, MapResult from .base import Base, Context from .workflow import WorkflowCanvas @@ -779,8 +779,10 @@ def load(self, func_name): :raise Exception """ - with modified_environ(IDDS_IGNORE_WORK_DECORATOR='true'): - func = super(Work, self).load(func_name) + os.environ['IDDS_IWORKFLOW_LOAD_WORK'] = 'true' + func = super(Work, self).load(func_name) + del os.environ['IDDS_IWORKFLOW_LOAD_WORK'] + return func def pre_run(self): @@ -920,7 +922,7 @@ def work(func=None, *, map_results=False, lazy=False, init_env=None): if func is None: return functools.partial(work, map_results=map_results, lazy=lazy, init_env=init_env) - if 'IDDS_IGNORE_WORK_DECORATOR' in os.environ: + if 'IDDS_IWORKFLOW_LOAD_WORK' in os.environ: return func @functools.wraps(func) diff --git a/workflow/lib/idds/iworkflow/workflow.py b/workflow/lib/idds/iworkflow/workflow.py index 5d1ee912..cf4eb719 100644 --- a/workflow/lib/idds/iworkflow/workflow.py +++ b/workflow/lib/idds/iworkflow/workflow.py @@ -23,8 +23,7 @@ # from idds.common import exceptions from idds.common.constants import WorkflowType -from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64,\ - modified_environ +from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64 from .asyncresult import AsyncResult from .base import Base, Context @@ -639,7 +638,9 @@ def get_idds_env(self): def get_idds_server(self): if 'IDDS_HOST' in self._idds_env: return self._idds_env['IDDS_HOST'] - return os.environ.get('IDDS_HOST', None) + if os.environ.get('IDDS_HOST', None): + return os.environ.get('IDDS_HOST', None) + return None def prepare_with_idds(self): """ @@ -1058,8 +1059,10 @@ def load(self, func_name): :raise Exception """ - with modified_environ(IDDS_IGNORE_WORKFLOW_DECORATOR='true'): - func = super(Workflow, self).load(func_name) + os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] = 'true' + func = super(Workflow, self).load(func_name) + del os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] + return func def pre_run(self): @@ -1158,7 +1161,7 @@ def workflow(func=None, *, local=False, service='idds', source_dir=None, primary return functools.partial(workflow, local=local, service=service, source_dir=source_dir, primary=primary, queue=queue, site=site, cloud=cloud, max_walltime=max_walltime, distributed=distributed, init_env=init_env) - if 'IDDS_IGNORE_WORKFLOW_DECORATOR' in os.environ: + if 'IDDS_IWORKFLOW_LOAD_WORKFLOW' in os.environ: return func @functools.wraps(func) From 032cf84193a9a30a65c18e1b47d80a003d8b84dc Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 15 Apr 2024 10:19:35 +0200 Subject: [PATCH 6/7] update setup for dev --- main/tools/env/setup_panda.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/main/tools/env/setup_panda.sh b/main/tools/env/setup_panda.sh index a379dbea..7d7bc097 100644 --- a/main/tools/env/setup_panda.sh +++ b/main/tools/env/setup_panda.sh @@ -56,6 +56,37 @@ elif [ "$instance" == "usdf" ]; then # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ export PANDA_CONFIG_ROOT=~/.panda/ +elif [ "$instance" == "new" ]; then + export PANDA_AUTH=oidc + export PANDA_URL_SSL=https://ai-idds-05.cern.ch:25443/server/panda + export PANDA_URL=http://ai-idds-05.cern.ch:25080/server/panda + export PANDAMON_URL=https://panda-doma.cern.ch + # export PANDA_AUTH_VO=panda_dev + export PANDA_AUTH_VO=Rubin:production + + export PANDACACHE_URL=$PANDA_URL_SSL + + export PANDA_SYS=/afs/cern.ch/user/w/wguan/workdisk/iDDS/.conda/iDDS/ + # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ + export PANDA_CONFIG_ROOT=~/.panda/ + + # export IDDS_HOST=https://aipanda015.cern.ch:443/idds + + # dev + # export IDDS_HOST=https://aipanda104.cern.ch:443/idds + + # doma + export IDDS_HOST=https://aipanda105.cern.ch:443/idds + + # export IDDS_BROKERS=atlas-test-mb.cern.ch:61013 + # export IDDS_BROKER_DESTINATION=/topic/doma.idds + # export IDDS_BROKER_USERNAME=domaidds + # export IDDS_BROKER_PASSWORD=1d25yeft6krJ1HFH + # export IDDS_BROKER_TIMEOUT=360 + + PANDA_QUEUE=BNL_OSG_2 + PANDA_WORKING_GROUP=EIC + PANDA_VO=wlcg else export PANDA_AUTH=oidc export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda From f589e03537739ba53002248d60a97b52a9bd6d58 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 15 Apr 2024 12:44:05 +0200 Subject: [PATCH 7/7] update env --- common/lib/idds/common/utils.py | 32 ++++++++++++++++++++++++- monitor/data/conf.js | 12 +++++----- workflow/lib/idds/iworkflow/work.py | 10 ++++---- workflow/lib/idds/iworkflow/workflow.py | 14 +++++------ workflow/tools/env/environment.yml | 2 +- workflow/tools/make/environment.yaml | 2 +- 6 files changed, 50 insertions(+), 22 deletions(-) diff --git a/common/lib/idds/common/utils.py b/common/lib/idds/common/utils.py index 1a73cde2..87bcdab0 100644 --- a/common/lib/idds/common/utils.py +++ b/common/lib/idds/common/utils.py @@ -7,9 +7,10 @@ # # Authors: # - Wen Guan, , 2019 - 2024 - +# - Lino Oscar Gerlach, , 2024 import base64 +import contextlib import errno import datetime import importlib @@ -942,3 +943,32 @@ def idds_mask(dict_): else: ret[k] = dict_[k] return ret + + +@contextlib.contextmanager +def modified_environ(*remove, **update): + """ + Temporarily updates the ``os.environ`` dictionary in-place. + The ``os.environ`` dictionary is updated in-place so that the modification + is sure to work in all situations. + :param remove: Environment variables to remove. + :param update: Dictionary of environment variables and values to add/update. + """ + env = os.environ + update = update or {} + remove = remove or [] + + # List of environment variables being updated or removed. + stomped = (set(update.keys()) | set(remove)) & set(env.keys()) + # Environment variables and values to restore on exit. + update_after = {k: env[k] for k in stomped} + # Environment variables and values to remove on exit. + remove_after = frozenset(k for k in update if k not in env) + + try: + env.update(update) + [env.pop(k, None) for k in remove] + yield + finally: + env.update(update_after) + [env.pop(k) for k in remove_after] diff --git a/monitor/data/conf.js b/monitor/data/conf.js index dd4ac352..c5e4e602 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus926.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus926.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus926.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus935.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus935.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus935.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/workflow/lib/idds/iworkflow/work.py b/workflow/lib/idds/iworkflow/work.py index f15e9cb3..55e47e91 100644 --- a/workflow/lib/idds/iworkflow/work.py +++ b/workflow/lib/idds/iworkflow/work.py @@ -7,6 +7,7 @@ # # Authors: # - Wen Guan, , 2023 - 2024 +# - Lino Oscar Gerlach, , 2024 import base64 import datetime @@ -21,7 +22,7 @@ from idds.common import exceptions from idds.common.constants import WorkflowType, TransformStatus from idds.common.imports import get_func_name -from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64 +from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64, modified_environ from .asyncresult import AsyncResult, MapResult from .base import Base, Context from .workflow import WorkflowCanvas @@ -779,9 +780,8 @@ def load(self, func_name): :raise Exception """ - os.environ['IDDS_IWORKFLOW_LOAD_WORK'] = 'true' - func = super(Work, self).load(func_name) - del os.environ['IDDS_IWORKFLOW_LOAD_WORK'] + with modified_environ(IDDS_IGNORE_WORK_DECORATOR='true'): + func = super(Work, self).load(func_name) return func @@ -922,7 +922,7 @@ def work(func=None, *, map_results=False, lazy=False, init_env=None): if func is None: return functools.partial(work, map_results=map_results, lazy=lazy, init_env=init_env) - if 'IDDS_IWORKFLOW_LOAD_WORK' in os.environ: + if 'IDDS_IGNORE_WORK_DECORATOR' in os.environ: return func @functools.wraps(func) diff --git a/workflow/lib/idds/iworkflow/workflow.py b/workflow/lib/idds/iworkflow/workflow.py index cf4eb719..9d718ae5 100644 --- a/workflow/lib/idds/iworkflow/workflow.py +++ b/workflow/lib/idds/iworkflow/workflow.py @@ -7,6 +7,7 @@ # # Authors: # - Wen Guan, , 2023 - 2024 +# - Lino Oscar Gerlach, , 2024 import base64 import collections @@ -23,7 +24,7 @@ # from idds.common import exceptions from idds.common.constants import WorkflowType -from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64 +from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64, modified_environ from .asyncresult import AsyncResult from .base import Base, Context @@ -638,9 +639,7 @@ def get_idds_env(self): def get_idds_server(self): if 'IDDS_HOST' in self._idds_env: return self._idds_env['IDDS_HOST'] - if os.environ.get('IDDS_HOST', None): - return os.environ.get('IDDS_HOST', None) - return None + return os.environ.get('IDDS_HOST', None) def prepare_with_idds(self): """ @@ -1059,9 +1058,8 @@ def load(self, func_name): :raise Exception """ - os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] = 'true' - func = super(Workflow, self).load(func_name) - del os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] + with modified_environ(IDDS_IGNORE_WORKFLOW_DECORATOR='true'): + func = super(Workflow, self).load(func_name) return func @@ -1161,7 +1159,7 @@ def workflow(func=None, *, local=False, service='idds', source_dir=None, primary return functools.partial(workflow, local=local, service=service, source_dir=source_dir, primary=primary, queue=queue, site=site, cloud=cloud, max_walltime=max_walltime, distributed=distributed, init_env=init_env) - if 'IDDS_IWORKFLOW_LOAD_WORKFLOW' in os.environ: + if 'IDDS_IGNORE_WORKFLOW_DECORATOR' in os.environ: return func @functools.wraps(func) diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index b59b2e54..c2ba6e1a 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -5,5 +5,5 @@ dependencies: - pip: - anytree - networkx - - stomp.py==8.0.1 + - stomp.py<=8.0.1 - idds-common==2.0.9 diff --git a/workflow/tools/make/environment.yaml b/workflow/tools/make/environment.yaml index f1c8273a..4c78df39 100644 --- a/workflow/tools/make/environment.yaml +++ b/workflow/tools/make/environment.yaml @@ -11,7 +11,7 @@ dependencies: - packaging - anytree - networkx - - stomp.py==8.0.1 + - stomp.py<=8.0.1 - panda-client - cffi - charset_normalizer