diff --git a/meta.py b/meta.py index 6c61e4625..829f60dfa 100644 --- a/meta.py +++ b/meta.py @@ -377,68 +377,138 @@ def update_index_metadata(ctx, path, metadata, creation_time, data_package): """Update the index attributes for JSON metadata.""" msi.coll_create(ctx, path, "", irods_types.BytesBuf()) ctx.msi_rmw_avu('-C', path, '%', '%', constants.UUFLATINDEX) + avu_op = "add" + avu_unit = constants.UUFLATINDEX + metadata_operations = { + "entity_name": path, + "entity_type": "collection", + "operations": [] + } for creator in metadata['Creator']: name = creator['Name'] if 'Given_Name' in name and 'Family_Name' in name: - ctx.msi_add_avu('-C', path, 'Creator', - name['Given_Name'] + ' ' + name['Family_Name'], - constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Creator", + "value": name['Given_Name'] + ' ' + name['Family_Name'], + "units": avu_unit + }) if 'Owner_Role' in creator: - ctx.msi_add_avu('-C', path, 'Owner_Role', creator['Owner_Role'], - constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Owner_Role", + "value": creator['Owner_Role'], + "units": avu_unit + }) if 'Contributor' in metadata: for contributor in metadata['Contributor']: if 'Name' in contributor: name = contributor['Name'] if 'Given_Name' in name and 'Family_Name' in name: - ctx.msi_add_avu('-C', path, 'Contributor', - name['Given_Name'] + ' ' + name['Family_Name'], - constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Contributor", + "value": name['Given_Name'] + ' ' + name['Family_Name'], + "units": avu_unit + }) if 'Tag' in metadata: for tag in metadata['Tag']: - ctx.msi_add_avu('-C', path, 'Tag', tag, - constants.UUFLATINDEX) - - ctx.msi_add_avu('-C', path, 'Title', metadata['Title'], - constants.UUFLATINDEX) - ctx.msi_add_avu('-C', path, 'Description', metadata['Description'], - constants.UUFLATINDEX) - ctx.msi_add_avu('-C', path, 'Data_Access_Restriction', - metadata['Data_Access_Restriction'], constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Tag", + "value": tag, + "units": avu_unit + }) + + extend_operations = [ + { + "operation": avu_op, + "attribute": "Title", + "value": metadata['Title'], + "units": avu_unit + }, + { + "operation": avu_op, + "attribute": "Description", + "value": metadata['Description'], + "units": avu_unit + }, + { + "operation": avu_op, + "attribute": "Data_Access_Restriction", + "value": metadata['Data_Access_Restriction'], + "units": avu_unit + }, + { + "operation": avu_op, + "attribute": "Creation_Time", + "value": creation_time, + "units": avu_unit + }, + { + "operation": avu_op, + "attribute": "Creation_Year", + "value": str(datetime.fromtimestamp(int(creation_time)).year), + "units": avu_unit + }] + + metadata_operations['operations'].extend(extend_operations) + if 'Research_Group' in metadata: - ctx.msi_add_avu('-C', path, 'Research_Group', - metadata['Research_Group'], constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Research_Group", + "value": metadata['Research_Group'], + "units": avu_unit + }) if 'Collection_Name' in metadata: - ctx.msi_add_avu('-C', path, 'Collection_Name', - metadata['Collection_Name'], constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Collection_Name", + "value": metadata['Collection_Name'], + "units": avu_unit + }) if 'Collected' in metadata: if 'Start_Date' in metadata['Collected']: - ctx.msi_add_avu('-C', path, 'Collected_Start_Year', - metadata['Collected']['Start_Date'][:4], - constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Collected_Start_Year", + "value": metadata['Collected']['Start_Date'][:4], + "units": avu_unit + }) if 'End_Date' in metadata['Collected']: - ctx.msi_add_avu('-C', path, 'Collected_End_Year', - metadata['Collected']['End_Date'][:4], - constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Collected_End_Year", + "value": metadata['Collected']['End_Date'][:4], + "units": avu_unit + }) if 'GeoLocation' in metadata: for geoLocation in metadata['GeoLocation']: if 'Description_Spatial' in geoLocation: - ctx.msi_add_avu('-C', path, 'Description_Spatial', geoLocation['Description_Spatial'], - constants.UUFLATINDEX) - - ctx.msi_add_avu('-C', path, 'Creation_Time', creation_time, - constants.UUFLATINDEX) - ctx.msi_add_avu('-C', path, 'Creation_Year', - str(datetime.fromtimestamp(int(creation_time)).year), - constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": avu_op, + "attribute": "Description_Spatial", + "value": geoLocation['Description_Spatial'], + "units": avu_unit + }) if config.enable_data_package_reference: - ctx.msi_add_avu('-C', path, 'Data_Package_Reference', data_package, - constants.UUFLATINDEX) + metadata_operations['operations'].append({ + "operation": "add", + "attribute": "Data_Package_Reference", + "value": data_package, + "units": avu_unit + }) + + if avu.apply_atomic_operations(ctx, metadata_operations): + log.write(ctx, 'update_index_metadata: Metadata index update successful on path {}'.format(path)) + else: + log.write(ctx, 'update_index_metadata: Metadata index update unsuccessful on path {}'.format(path)) def ingest_metadata_vault(ctx, path): diff --git a/replication.py b/replication.py index 83a233b81..eac2e93d1 100644 --- a/replication.py +++ b/replication.py @@ -46,7 +46,19 @@ def replicate_asynchronously(ctx, path, source_resource, target_resource): if not already_has_avu: # Can't use mod_avu/set here (instead of add_avu) because it would be blocked by metadata policies. - msi.add_avu(ctx, '-d', path, replication_avu_name, replication_avu_value, "") + add_operation = { + "entity_name": path, + "entity_type": "data_object", + "operations": [ + { + "operation": "add", + "attribute": replication_avu_name, + "value": replication_avu_value, + "units": "" + } + ] + } + avu.apply_atomic_operations(ctx, add_operation) except msi.Error as e: if "-817000" in str(e): # CAT_UNKNOWN_FILE: object has been removed in the mean time. No need to replicate it anymore. @@ -132,7 +144,19 @@ def rule_replicate_batch(ctx, verbose, balance_id_min, balance_id_max, batch_siz # Not replicable. log.write(ctx, "ERROR - Invalid replication data for {}".format(path)) try: - ctx.msi_add_avu('-d', path, errorattr, "Invalid,Invalid", "") + add_operation = { + "entity_name": path, + "entity_type": "data_object", + "operations": [ + { + "operation": "add", + "attribute": errorattr, + "value": "Invalid,Invalid", + "units": "" + } + ] + } + avu.apply_atomic_operations(ctx, add_operation) except Exception: pass @@ -184,7 +208,19 @@ def rule_replicate_batch(ctx, verbose, balance_id_min, balance_id_max, batch_siz except msi.Error as e: log.write(ctx, 'ERROR - The file could not be replicated: {}'.format(str(e))) try: - ctx.msi_add_avu('-d', path, errorattr, "{},{}".format(from_path, to_path), "") + add_operation = { + "entity_name": path, + "entity_type": "data_object", + "operations": [ + { + "operation": "add", + "attribute": errorattr, + "value": "{},{}".format(from_path, to_path), + "units": "" + } + ] + } + avu.apply_atomic_operations(ctx, add_operation) except Exception: pass diff --git a/revisions.py b/revisions.py index da1704af1..07553652b 100644 --- a/revisions.py +++ b/revisions.py @@ -288,7 +288,19 @@ def resource_modified_post_revision(ctx, resource, zone, path): offset=0, limit=1, output=genquery.AS_LIST))) > 0 if not already_has_avu: - msi.add_avu(ctx, '-d', path, revision_avu_name, revision_avu_value, "") + add_operation = { + "entity_name": path, + "entity_type": "data_object", + "operations": [ + { + "operation": "add", + "attribute": revision_avu_name, + "value": revision_avu_value, + "units": "" + } + ] + } + avu.apply_atomic_operations(ctx, add_operation) except msi.Error as e: if "-817000" in str(e):