agent: remove bf:Agent

Co-Authored-by: Peter Weber <[email protected]>
rero · Apr 24, 2024 · 85158d1 · 85158d1
1 parent 71ac03d
commit 85158d1
Show file tree

Hide file tree

Showing 28 changed files with 8,150 additions and 16,085 deletions.
diff --git a/data/aggnd.json b/data/aggnd.json
diff --git a/data/aggnd_metadata.csv b/data/aggnd_metadata.csv
diff --git a/data/agrero.json b/data/agrero.json
diff --git a/data/agrero_metadata.csv b/data/agrero_metadata.csv
diff --git a/data/aidref.json b/data/aidref.json
diff --git a/data/aidref_metadata.csv b/data/aidref_metadata.csv
diff --git a/poetry.lock b/poetry.lock
diff --git a/rero_mef/agents/gnd/mappings/v7/agents_gnd/gnd-agent-v0.0.1.json b/rero_mef/agents/gnd/mappings/v7/agents_gnd/gnd-agent-v0.0.1.json
@@ -14,9 +14,6 @@
       "pid": {
         "type": "keyword"
       },
-      "bf:Agent": {
-        "type": "keyword"
-      },
       "type": {
         "type": "keyword"
       },

diff --git a/rero_mef/agents/idref/mappings/v7/agents_idref/idref-agent-v0.0.1.json b/rero_mef/agents/idref/mappings/v7/agents_idref/idref-agent-v0.0.1.json
@@ -14,9 +14,6 @@
       "pid": {
         "type": "keyword"
       },
-      "bf:Agent": {
-        "type": "keyword"
-      },
       "type": {
         "type": "keyword"
       },

diff --git a/rero_mef/agents/mef/api.py b/rero_mef/agents/mef/api.py
@@ -192,12 +192,6 @@ def get_latest(cls, pid_type, pid):
                 if search.count() > 0:
                     new_data = next(search.scan()).to_dict()
                     new_pid = new_data.get('idref', {}).get('pid')
-            for agent in cls.entities:
-                if agent_data := data.get(agent):
-                    if not agent_data.get('type'):
-                        data[agent]['type'] = agent_data['bf:Agent']
-                        if not data.get('type'):
-                            data['type'] = agent_data['bf:Agent']
             return cls.get_latest(pid_type=pid_type, pid=new_pid) \
                 if new_pid else data
         return {}

diff --git a/rero_mef/agents/mef/mappings/v7/mef/mef-v0.0.1.json b/rero_mef/agents/mef/mappings/v7/mef/mef-v0.0.1.json
@@ -56,9 +56,6 @@
           "pid": {
             "type": "keyword"
           },
-          "bf:Agent": {
-            "type": "keyword"
-          },
           "type": {
             "type": "keyword"
           },
@@ -164,9 +161,6 @@
           "pid": {
             "type": "keyword"
           },
-          "bf:Agent": {
-            "type": "keyword"
-          },
           "type": {
             "type": "keyword"
           },
@@ -272,9 +266,6 @@
           "pid": {
             "type": "keyword"
           },
-          "bf:Agent": {
-            "type": "keyword"
-          },
           "type": {
             "type": "keyword"
           },

diff --git a/rero_mef/agents/rero/mappings/v7/agents_rero/rero-agent-v0.0.1.json b/rero_mef/agents/rero/mappings/v7/agents_rero/rero-agent-v0.0.1.json
@@ -14,9 +14,6 @@
       "pid": {
         "type": "keyword"
       },
-      "bf:Agent": {
-        "type": "keyword"
-      },
       "type": {
         "type": "keyword"
       },

diff --git a/rero_mef/agents/serializers.py b/rero_mef/agents/serializers.py
@@ -65,10 +65,6 @@ def serialize(self, pid, record, links_factory=None, **kwargs):
         :param record: Record instance.
         :param links_factory: Factory function for record links.
         """
-        # TODO: could be deleted in the future
-        if not record.get('type'):
-            record['type'] = record['bf:Agent']
-
         return super(ReroMefSerializer, self).serialize(
             pid=pid, record=record, links_factory=add_links, **kwargs)
 

diff --git a/rero_mef/alembic/28982ef7ce63_delete_bf_agent.py b/rero_mef/alembic/28982ef7ce63_delete_bf_agent.py
@@ -0,0 +1,78 @@
+#
+# This file is part of Invenio.
+# Copyright (C) 2016-2018 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""delete bf:Agent."""
+
+import click
+from invenio_db import db
+from sqlalchemy import func
+
+from rero_mef.utils import get_entity_class
+
+# revision identifiers, used by Alembic.
+revision = '28982ef7ce63'
+down_revision = '0d0b3009dbe0'
+branch_labels = ()
+depends_on = None
+
+
+agents = {
+    'aggnd': 'GND',
+    'aidref': 'IDREF',
+    'agrero': 'RERO'
+}
+
+
+def commit_changes(indexer, ids):
+    """Commit changes and reindex."""
+    db.session.commit()
+    indexer.bulk_index(ids)
+    indexer.process_bulk_queue()
+
+
+def upgrade():
+    """Upgrade database."""
+    for agent, name in agents.items():
+        agent_class = get_entity_class(agent)
+
+        count = agent_class.model_cls.query.filter(
+            agent_class.model_cls.json.op('->')('bf:Agent') is not None
+        ).update(
+            {
+                agent_class.model_cls.json:
+                agent_class.model_cls.json - "bf:Agent"
+            },
+            synchronize_session=False
+        )
+        click.echo(
+            f'Delete bf:Agent from {agent}: {count}')
+        db.session.commit()
+    # For ES: update mapping must be done.
+
+
+def downgrade():
+    """Downgrade database."""
+    for agent, name in agents.items():
+        agent_class = get_entity_class(agent)
+
+        count = agent_class.model_cls.query.filter(
+            agent_class.model_cls.json.op('->')('type') is not None
+        ).update(
+            {
+                "json": func.jsonb_set(
+                    agent_class.model_cls.json,
+                    '{bf:Agent}',
+                    agent_class.model_cls.json.op('->')('type')
+                )
+            },
+            synchronize_session=False
+        )
+        click.echo(
+            f'Add bf:Agent to {agent}: {count}')
+        db.session.commit()
+    # For ES: update mapping must be done
+    # and all agents and MEF have to be reindexed
diff --git a/rero_mef/api.py b/rero_mef/api.py
@@ -137,7 +137,7 @@ def create_or_update(cls, data, id_=None, delete_pid=True, dbcommit=False,
             data = add_schema(data, agent_record.provider.pid_type)
             # save the records old data if the new one is empty
             copy_fields = ['pid', '$schema', 'identifier', 'identifiedBy',
-                           'authorized_access_point', 'type', 'bf:Agent',
+                           'authorized_access_point', 'type',
                            'relation_pid', 'deleted']
             original_data = {
                 k: v for k, v in agent_record.items() if k in copy_fields}

diff --git a/rero_mef/jsonschemas/common/agent-v0.0.1.json b/rero_mef/jsonschemas/common/agent-v0.0.1.json
@@ -22,14 +22,6 @@
       "type": "string",
       "minLength": 1
     },
-    "bf:Agent": {
-      "title": "Type",
-      "type": "string",
-      "enum": [
-        "bf:Person",
-        "bf:Organisation"
-      ]
-    },
     "type": {
       "title": "Type",
       "type": "string",

diff --git a/rero_mef/marctojson/do_gnd_agent.py b/rero_mef/marctojson/do_gnd_agent.py
@@ -368,7 +368,6 @@ def trans_gnd_authorized_access_point(self):
                 subfields=subfields
             )
             for authorized_access_point in authorized_access_points:
-                self.json_dict['bf:Agent'] = agent
                 self.json_dict['type'] = agent
                 if self.json_dict.get('authorized_access_point'):
                     variant_access_points.append(authorized_access_point)

diff --git a/rero_mef/marctojson/do_idref_agent.py b/rero_mef/marctojson/do_idref_agent.py
@@ -388,7 +388,6 @@ def trans_idref_authorized_access_point(self):
             tag_grouping=tag_grouping
         )
         for authorized_access_point in authorized_access_points:
-            self.json_dict['bf:Agent'] = agent
             self.json_dict['type'] = agent
             if self.json_dict.get('authorized_access_point'):
                 variant_access_points.append(authorized_access_point)

diff --git a/rero_mef/marctojson/do_rero_agent.py b/rero_mef/marctojson/do_rero_agent.py
@@ -192,7 +192,6 @@ def trans_rero_authorized_access_point(self):
         if self.logger and self.verbose:
             self.logger.info(
                 'Call Function', 'trans_rero_authorized_access_point')
-        self.json_dict['bf:Agent'] = agent
         self.json_dict['type'] = agent
         authorized_access_points = build_string_list_from_fields(
             record=self.marc,

diff --git a/rero_mef/utils.py b/rero_mef/utils.py
@@ -465,7 +465,11 @@ def oai_save_records_from_dates(name, file_name, sickle, oai_item_iterator,
                         rec.leader = f'{rec.leader[:9]}a{rec.leader[10:]}'
                         output_file.write(rec.as_marc())
                 except NoRecordsMatch:
+<<<<<<< HEAD
                     from_date = until_date
+=======
+                    from_date = from_date + timedelta(days=days_span)
+>>>>>>> f902a45 (agent: remove bf:Agent)
                     continue
                 except Exception as err:
                     current_app.logger.error(err)
@@ -475,7 +479,11 @@ def oai_save_records_from_dates(name, file_name, sickle, oai_item_iterator,
                         f'{from_date.strftime(TIME_FORMAT)} .. '
                         f'{until_date.strftime(TIME_FORMAT)}'
                     )
+<<<<<<< HEAD
                 from_date = until_date
+=======
+                from_date = from_date + timedelta(days=days_span)
+>>>>>>> f902a45 (agent: remove bf:Agent)
     if verbose:
         click.echo(f'OAI {name}: {count}')
     return count