diff --git a/sonar/monitoring/api/data_integrity.py b/sonar/monitoring/api/data_integrity.py index 61346eb7..5c401ff6 100644 --- a/sonar/monitoring/api/data_integrity.py +++ b/sonar/monitoring/api/data_integrity.py @@ -27,20 +27,19 @@ class DataIntegrityMonitoring(): """Data integrity monitoring.""" - def get_db_count(self, doc_type, with_deleted=False): + def get_db_count(self, rec_type, with_deleted=False): """Get database count. Get count of items in the database for the given document type. - :param doc_type: Resource type. + :param rec_type: Record type. :param with_deleted: Count also deleted items. :returns: Items count. """ - if not sonar.endpoints.get(doc_type) or doc_type == 'proj': - raise Exception( - 'No endpoint configured for "{type}"'.format(type=doc_type)) + if service := sonar.service(rec_type): + rec_type = service.record_cls.pid_type - query = PersistentIdentifier.query.filter_by(pid_type=doc_type) + query = PersistentIdentifier.query.filter_by(pid_type=rec_type) if not with_deleted: query = query.filter_by(status=PIDStatus.REGISTERED) @@ -59,29 +58,35 @@ def get_es_count(self, index): except NotFoundError: raise Exception('No index found for "{type}"'.format(type=index)) - def missing_pids(self, doc_type, with_deleted=False): + def missing_pids(self, rec_type, with_deleted=False): """Get ES and DB counts. - :param doc_type: Resource type. + :param rec_type: Record type. :param with_deleted: Check also delete items in database. """ - index = sonar.endpoints.get(doc_type, None) + index = sonar.endpoints.get(rec_type, None) if not index: raise Exception('No index configured for resource "{type}"'.format( - type=doc_type)) + type=rec_type)) result = {'es': [], 'es_double': [], 'db': []} # Elastic search PIDs es_pids = {} - for hit in RecordsSearch(index=index).source('pid').scan(): - if es_pids.get(hit.pid): + for hit in RecordsSearch(index=index).source(['pid', 'id']).scan(): + pid_value = hit.pid + # for resources pid is a dict + if not isinstance(pid_value, str): + pid_value = hit.id + if es_pids.get(pid_value): result['es_double'].append(hit.pid) - es_pids[hit.pid] = 1 + es_pids[pid_value] = 1 # Database PIDs - query = PersistentIdentifier.query.filter_by(pid_type=doc_type) + if service := sonar.service(rec_type): + rec_type = service.record_cls.pid_type + query = PersistentIdentifier.query.filter_by(pid_type=rec_type) if not with_deleted: query = query.filter_by(status=PIDStatus.REGISTERED) @@ -105,11 +110,11 @@ def info(self, with_deleted=False, with_detail=False): """ info = {} - for doc_type, index in sonar.endpoints.items(): + for rec_type, index in sonar.endpoints.items(): es_count = self.get_es_count(index) - db_count = self.get_db_count(doc_type) + db_count = self.get_db_count(rec_type) - info[doc_type] = { + info[rec_type] = { 'db': db_count, 'es': es_count, 'db-es': db_count - es_count, @@ -117,8 +122,8 @@ def info(self, with_deleted=False, with_detail=False): } if with_detail: - info[doc_type]['detail'] = self.missing_pids( - doc_type, with_deleted) + info[rec_type]['detail'] = self.missing_pids( + rec_type, with_deleted) return info @@ -128,7 +133,7 @@ def has_error(self, with_deleted=False): :param with_deleted: Count also deleted items in database. :returns: True if an error is found """ - for doc_type, item in self.info(with_deleted).items(): + for rec_type, item in self.info(with_deleted).items(): if item['db-es'] != 0: return True diff --git a/sonar/proxies.py b/sonar/proxies.py index 14a14c42..6d1fdfba 100644 --- a/sonar/proxies.py +++ b/sonar/proxies.py @@ -55,8 +55,7 @@ def service(self, resource_type): :returns: A service instance """ if not self.resources.get(resource_type): - raise Exception( - f'No service configured for resource "{resource_type}"') + return None return self.resources[resource_type].service diff --git a/tests/unit/monitoring/test_api_data_integrity.py b/tests/unit/monitoring/test_api_data_integrity.py index f9fdb188..07e421b7 100644 --- a/tests/unit/monitoring/test_api_data_integrity.py +++ b/tests/unit/monitoring/test_api_data_integrity.py @@ -29,9 +29,7 @@ def test_db_count(app, es_clear, document): monitoring = DataIntegrityMonitoring() # Resource not configured - with pytest.raises(Exception) as exception: - monitoring.get_db_count('not-existing') - assert str(exception.value) == 'No endpoint configured for "not-existing"' + assert not monitoring.get_db_count('not-existing') # OK assert monitoring.get_db_count('doc') == 1 @@ -111,16 +109,27 @@ def test_missing_pids(app, es_clear, document_json): 'RECORDS_REST_ENDPOINTS')['doc']['search_index'] = 'documents' -def test_info(app, es_clear, document, deposit): +def test_info(app, es_clear, document, project, deposit): """Test info.""" monitoring = DataIntegrityMonitoring() info = monitoring.info() - for doc_type in ['depo', 'doc', 'org', 'projects', 'user']: - assert doc_type in info + for rec_type in ['depo', 'doc', 'org', 'projects', 'user']: + assert rec_type in info # With detail info = monitoring.info(with_detail=True) assert info['doc']['detail'] == {'db': [], 'es': [], 'es_double': []} + assert info['doc']['db-es'] == 0 + assert info['doc']['db'] == 1 + assert info['doc']['db'] == 1 + assert info['depo']['detail'] == {'db': [], 'es': [], 'es_double': []} + assert info['depo']['db-es'] == 0 + assert info['depo']['db'] == 1 + assert info['depo']['es'] == 1 + assert info['projects']['detail'] == {'db': [], 'es': [], 'es_double': []} + assert info['projects']['db-es'] == 0 + assert info['projects']['db'] == 1 + assert info['projects']['es'] == 1 def test_has_error(app, es_clear, document): diff --git a/tests/unit/test_sonar_proxy.py b/tests/unit/test_sonar_proxy.py index 7bb83591..1573cf7f 100644 --- a/tests/unit/test_sonar_proxy.py +++ b/tests/unit/test_sonar_proxy.py @@ -17,8 +17,6 @@ """Test SONAR proxy.""" -import pytest - from sonar.proxies import sonar from sonar.resources.projects.service import RecordService @@ -38,7 +36,4 @@ def test_service(app): service = sonar.service('projects') assert isinstance(service, RecordService) - with pytest.raises(Exception) as exception: - sonar.service('unknown') - assert str( - exception.value) == 'No service configured for resource "unknown"' + assert not sonar.service('unknown')