diff --git a/__init__.py b/__init__.py index fc16f3409..438312d66 100644 --- a/__init__.py +++ b/__init__.py @@ -29,6 +29,7 @@ # so that they become visible to iRODS. from admin import * +from arb import * from browse import * from folder import * from groups import * @@ -39,10 +40,10 @@ from meta_form import * from provenance import * from research import * -from resources import * from schema import * from schema_transformation import * from schema_transformations import * +from stats import * from publication_troubleshoot import * from vault import * from datacite import * diff --git a/arb.py b/arb.py new file mode 100644 index 000000000..a77949a7a --- /dev/null +++ b/arb.py @@ -0,0 +1,81 @@ +"""Functions for automatic resource balancing module.""" + +__copyright__ = 'Copyright (c) 2019-2024, Utrecht University' +__license__ = 'GPLv3, see LICENSE' + +from util import * + +__all__ = ['rule_resource_update_resc_arb_data', + 'rule_resource_update_misc_arb_data'] + + +@rule.make(inputs=[0, 1, 2], outputs=[]) +def rule_resource_update_resc_arb_data(ctx: rule.Context, resc_name: str, bytes_free: int, bytes_total: int) -> None: + """ + Update ARB data for a specific resource + + :param ctx: Combined type of a callback and rei struct + :param resc_name: Name of a particular unixfilesystem resource + :param bytes_free: Free size on this resource, in bytes + :param bytes_total: Total size of this resource, in bytes + """ + if user.user_type(ctx) != 'rodsadmin': + log.write(ctx, "Error: insufficient permissions to run ARB data update rule.") + return + + if not resources.exists(ctx, resc_name): + log.write(ctx, "Error: could not find resource named '{}' for ARB update.".format(resc_name)) + return + + bytes_free_gb = int(bytes_free) / 2 ** 30 + bytes_free_percent = 100 * (float(bytes_free) / float(bytes_total)) + + if resc_name in config.arb_exempt_resources: + arb_status = constants.arb_status.EXEMPT + elif bytes_free_gb >= config.arb_min_gb_free and bytes_free_percent > config.arb_min_percent_free: + arb_status = constants.arb_status.AVAILABLE + else: + arb_status = constants.arb_status.FULL + + parent_resc_name = resources.get_parent_by_name(ctx, resc_name) + + manager = arb_data_manager.ARBDataManager() + manager.put(ctx, resc_name, constants.arb_status.IGNORE) + + if parent_resc_name is not None and resources.get_type_by_name(ctx, parent_resc_name) == "passthru": + manager.put(ctx, parent_resc_name, arb_status) + + +@rule.make() +def rule_resource_update_misc_arb_data(ctx: rule.Context) -> None: + """Update ARB data for resources that are not covered by the regular process. That is, + all resources that are neither unixfilesystem nor passthrough resources, as well as + passthrough resources that do not have a unixfilesystem child resource. + + :param ctx: Combined type of a callback and rei struct + """ + if user.user_type(ctx) != 'rodsadmin': + log.write(ctx, "Error: insufficient permissions to run ARB data update rule.") + return + + manager = arb_data_manager.ARBDataManager() + + all_resources = resources.get_all_resource_names(ctx) + ufs_resources = set(resources.get_resource_names_by_type(ctx, "unixfilesystem") + + resources.get_resource_names_by_type(ctx, "unix file system")) + pt_resources = set(resources.get_resource_names_by_type(ctx, "passthru")) + + for resc in all_resources: + if resc in ufs_resources: + pass + elif resc not in pt_resources: + manager.put(ctx, resc, constants.arb_status.IGNORE) + else: + child_resources = resources.get_children_by_name(ctx, resc) + child_found = False + for child_resource in child_resources: + if child_resource in ufs_resources: + child_found = True + # Ignore the passthrough resource if it does not have a UFS child resource + if not child_found: + manager.put(ctx, resc, constants.arb_status.IGNORE) diff --git a/setup.cfg b/setup.cfg index 880b648c2..9ecaef0c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ strictness=short docstring_style=sphinx max-line-length=127 exclude=__init__.py,tools,tests/env/ -application-import-names=avu,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resource,yoda_names,policies_utils +application-import-names=arb,avu,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,stats,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resource,yoda_names,policies_utils [mypy] exclude = tools|unit-tests diff --git a/resources.py b/stats.py similarity index 89% rename from resources.py rename to stats.py index 6a496e9f1..66944ad82 100644 --- a/resources.py +++ b/stats.py @@ -556,78 +556,6 @@ def rule_resource_store_storage_statistics(ctx: rule.Context) -> str: return 'ok' -@rule.make(inputs=[0, 1, 2], outputs=[]) -def rule_resource_update_resc_arb_data(ctx: rule.Context, resc_name: str, bytes_free: int, bytes_total: int) -> None: - """ - Update ARB data for a specific resource - - :param ctx: Combined type of a callback and rei struct - :param resc_name: Name of a particular unixfilesystem resource - :param bytes_free: Free size on this resource, in bytes - :param bytes_total: Total size of this resource, in bytes - """ - if user.user_type(ctx) != 'rodsadmin': - log.write(ctx, "Error: insufficient permissions to run ARB data update rule.") - return - - if not resources.exists(ctx, resc_name): - log.write(ctx, "Error: could not find resource named '{}' for ARB update.".format(resc_name)) - return - - bytes_free_gb = int(bytes_free) / 2 ** 30 - bytes_free_percent = 100 * (float(bytes_free) / float(bytes_total)) - - if resc_name in config.arb_exempt_resources: - arb_status = constants.arb_status.EXEMPT - elif bytes_free_gb >= config.arb_min_gb_free and bytes_free_percent > config.arb_min_percent_free: - arb_status = constants.arb_status.AVAILABLE - else: - arb_status = constants.arb_status.FULL - - parent_resc_name = resources.get_parent_by_name(ctx, resc_name) - - manager = arb_data_manager.ARBDataManager() - manager.put(ctx, resc_name, constants.arb_status.IGNORE) - - if parent_resc_name is not None and resources.get_type_by_name(ctx, parent_resc_name) == "passthru": - manager.put(ctx, parent_resc_name, arb_status) - - -@rule.make() -def rule_resource_update_misc_arb_data(ctx: rule.Context) -> None: - """Update ARB data for resources that are not covered by the regular process. That is, - all resources that are neither unixfilesystem nor passthrough resources, as well as - passthrough resources that do not have a unixfilesystem child resource. - - :param ctx: Combined type of a callback and rei struct - """ - if user.user_type(ctx) != 'rodsadmin': - log.write(ctx, "Error: insufficient permissions to run ARB data update rule.") - return - - manager = arb_data_manager.ARBDataManager() - - all_resources = resources.get_all_resource_names(ctx) - ufs_resources = set(resources.get_resource_names_by_type(ctx, "unixfilesystem") - + resources.get_resource_names_by_type(ctx, "unix file system")) - pt_resources = set(resources.get_resource_names_by_type(ctx, "passthru")) - - for resc in all_resources: - if resc in ufs_resources: - pass - elif resc not in pt_resources: - manager.put(ctx, resc, constants.arb_status.IGNORE) - else: - child_resources = resources.get_children_by_name(ctx, resc) - child_found = False - for child_resource in child_resources: - if child_resource in ufs_resources: - child_found = True - # Ignore the passthrough resource if it does not have a UFS child resource - if not child_found: - manager.put(ctx, resc, constants.arb_status.IGNORE) - - def get_categories(ctx: rule.Context) -> List: """Get all categories for current user.