monitoring: fix projects database count

* Fixes the invenio records resources based resources: projects.

Co-Authored-by: Johnny Mariéthoz <[email protected]>

sonar/monitoring/api/data_integrity.py

@@ -27,20 +27,19 @@
 class DataIntegrityMonitoring():
     """Data integrity monitoring."""
 
-    def get_db_count(self, doc_type, with_deleted=False):
+    def get_db_count(self, rec_type, with_deleted=False):
         """Get database count.
 
         Get count of items in the database for the given document type.
 
-        :param doc_type: Resource type.
+        :param rec_type: Record type.
         :param with_deleted: Count also deleted items.
         :returns: Items count.
         """
-        if not sonar.endpoints.get(doc_type) or doc_type == 'proj':
-            raise Exception(
-                'No endpoint configured for "{type}"'.format(type=doc_type))
+        if service := sonar.service(rec_type):
+            rec_type = service.record_cls.pid_type
 
-        query = PersistentIdentifier.query.filter_by(pid_type=doc_type)
+        query = PersistentIdentifier.query.filter_by(pid_type=rec_type)
         if not with_deleted:
             query = query.filter_by(status=PIDStatus.REGISTERED)
 
@@ -59,29 +58,35 @@ def get_es_count(self, index):
         except NotFoundError:
             raise Exception('No index found for "{type}"'.format(type=index))
 
-    def missing_pids(self, doc_type, with_deleted=False):
+    def missing_pids(self, rec_type, with_deleted=False):
         """Get ES and DB counts.
 
-        :param doc_type: Resource type.
+        :param rec_type: Record type.
         :param with_deleted: Check also delete items in database.
         """
-        index = sonar.endpoints.get(doc_type, None)
+        index = sonar.endpoints.get(rec_type, None)
 
         if not index:
             raise Exception('No index configured for resource "{type}"'.format(
-                type=doc_type))
+                type=rec_type))
 
         result = {'es': [], 'es_double': [], 'db': []}
 
         # Elastic search PIDs
         es_pids = {}
-        for hit in RecordsSearch(index=index).source('pid').scan():
-            if es_pids.get(hit.pid):
+        for hit in RecordsSearch(index=index).source(['pid', 'id']).scan():
+            pid_value = hit.pid
+            # for resources pid is a dict
+            if not isinstance(pid_value, str):
+                pid_value = hit.id
+            if es_pids.get(pid_value):
                 result['es_double'].append(hit.pid)
-            es_pids[hit.pid] = 1
+            es_pids[pid_value] = 1
 
         # Database PIDs
-        query = PersistentIdentifier.query.filter_by(pid_type=doc_type)
+        if service := sonar.service(rec_type):
+            rec_type = service.record_cls.pid_type
+        query = PersistentIdentifier.query.filter_by(pid_type=rec_type)
         if not with_deleted:
             query = query.filter_by(status=PIDStatus.REGISTERED)
 
@@ -105,20 +110,20 @@ def info(self, with_deleted=False, with_detail=False):
         """
         info = {}
 
-        for doc_type, index in sonar.endpoints.items():
+        for rec_type, index in sonar.endpoints.items():
             es_count = self.get_es_count(index)
-            db_count = self.get_db_count(doc_type)
+            db_count = self.get_db_count(rec_type)
 
-            info[doc_type] = {
+            info[rec_type] = {
                 'db': db_count,
                 'es': es_count,
                 'db-es': db_count - es_count,
                 'index': index
             }
 
             if with_detail:
-                info[doc_type]['detail'] = self.missing_pids(
-                    doc_type, with_deleted)
+                info[rec_type]['detail'] = self.missing_pids(
+                    rec_type, with_deleted)
 
         return info
 
@@ -128,7 +133,7 @@ def has_error(self, with_deleted=False):
         :param with_deleted: Count also deleted items in database.
         :returns: True if an error is found
         """
-        for doc_type, item in self.info(with_deleted).items():
+        for rec_type, item in self.info(with_deleted).items():
             if item['db-es'] != 0:
                 return True
 

sonar/proxies.py

@@ -55,8 +55,7 @@ def service(self, resource_type):
         :returns: A service instance
         """
         if not self.resources.get(resource_type):
-            raise Exception(
-                f'No service configured for resource "{resource_type}"')
+            return None
 
         return self.resources[resource_type].service
 

tests/unit/monitoring/test_api_data_integrity.py

@@ -29,9 +29,7 @@ def test_db_count(app, es_clear, document):
     monitoring = DataIntegrityMonitoring()
 
     # Resource not configured
-    with pytest.raises(Exception) as exception:
-        monitoring.get_db_count('not-existing')
-    assert str(exception.value) == 'No endpoint configured for "not-existing"'
+    assert not monitoring.get_db_count('not-existing')
 
     # OK
     assert monitoring.get_db_count('doc') == 1
@@ -111,16 +109,27 @@ def test_missing_pids(app, es_clear, document_json):
         'RECORDS_REST_ENDPOINTS')['doc']['search_index'] = 'documents'
 
 
-def test_info(app, es_clear, document, deposit):
+def test_info(app, es_clear, document, project, deposit):
     """Test info."""
     monitoring = DataIntegrityMonitoring()
     info = monitoring.info()
-    for doc_type in ['depo', 'doc', 'org', 'projects', 'user']:
-        assert doc_type in info
+    for rec_type in ['depo', 'doc', 'org', 'projects', 'user']:
+        assert rec_type in info
 
     # With detail
     info = monitoring.info(with_detail=True)
     assert info['doc']['detail'] == {'db': [], 'es': [], 'es_double': []}
+    assert info['doc']['db-es'] == 0
+    assert info['doc']['db'] == 1
+    assert info['doc']['db'] == 1
+    assert info['depo']['detail'] == {'db': [], 'es': [], 'es_double': []}
+    assert info['depo']['db-es'] == 0
+    assert info['depo']['db'] == 1
+    assert info['depo']['es'] == 1
+    assert info['projects']['detail'] == {'db': [], 'es': [], 'es_double': []}
+    assert info['projects']['db-es'] == 0
+    assert info['projects']['db'] == 1
+    assert info['projects']['es'] == 1
 
 
 def test_has_error(app, es_clear, document):

tests/unit/test_sonar_proxy.py

@@ -17,8 +17,6 @@
 
 """Test SONAR proxy."""
 
-import pytest
-
 from sonar.proxies import sonar
 from sonar.resources.projects.service import RecordService
 
@@ -38,7 +36,4 @@ def test_service(app):
     service = sonar.service('projects')
     assert isinstance(service, RecordService)
 
-    with pytest.raises(Exception) as exception:
-        sonar.service('unknown')
-    assert str(
-        exception.value) == 'No service configured for resource "unknown"'
+    assert not sonar.service('unknown')