diff --git a/docker/wis2-gdc.env b/docker/wis2-gdc.env index aca4791..b548fec 100644 --- a/docker/wis2-gdc.env +++ b/docker/wis2-gdc.env @@ -6,6 +6,7 @@ export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@mosquitto:1883 export WIS2_GDC_CENTRE_ID=ca-eccc-msc-gdc export WIS2_GDC_GB=mqtt://everyone:everyone@mosquitto:1883 export WIS2_GDC_GB_TOPIC=origin/a/wis2/+/+/metadata/# +export WIS2_GDC_OPENMETRICS_FILE=/tmp/wis2-gdc-openmetrics.txt export WIS2_GDC_PUBLISH_REPORTS=true # global broker links diff --git a/wis2-gdc.env b/wis2-gdc.env index 9b5dfa5..4aa886c 100644 --- a/wis2-gdc.env +++ b/wis2-gdc.env @@ -6,6 +6,7 @@ export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@localhost:1883 export WIS2_GDC_CENTRE_ID=ca-eccc-msc-gdc export WIS2_GDC_GB=mqtt://everyone:everyone@localhost:1883 export WIS2_GDC_GB_TOPIC=origin/a/wis2/+/+/metadata/# +export WIS2_GDC_OPENMETRICS_FILE=/tmp/wis2-gdc-openmetrics.txt export WIS2_GDC_PUBLISH_REPORTS=true # global broker links diff --git a/wis2_gdc/__init__.py b/wis2_gdc/__init__.py index e2b1483..e7fb0ec 100644 --- a/wis2_gdc/__init__.py +++ b/wis2_gdc/__init__.py @@ -22,6 +22,7 @@ import click from wis2_gdc.registrar import register, setup, teardown +from wis2_gdc.monitor import monitor from wis2_gdc.sync import archive, sync __version__ = '0.1.dev0' @@ -35,8 +36,9 @@ def cli(): pass -cli.add_command(archive) -cli.add_command(sync) -cli.add_command(register) cli.add_command(setup) cli.add_command(teardown) +cli.add_command(register) +cli.add_command(sync) +cli.add_command(monitor) +cli.add_command(archive) diff --git a/wis2_gdc/env.py b/wis2_gdc/env.py index e28d9a5..eddbf24 100644 --- a/wis2_gdc/env.py +++ b/wis2_gdc/env.py @@ -49,12 +49,16 @@ def str2bool(value: Any) -> bool: BACKEND_CONNECTION = os.environ.get('WIS2_GDC_BACKEND_CONNECTION') BROKER_URL = os.environ.get('WIS2_GDC_BROKER_URL') CENTRE_ID = os.environ.get('WIS2_GDC_CENTRE_ID') +GB = os.environ.get('WIS2_GDC_GB') +GB_TOPIC = os.environ.get('WIS2_GDC_GB_TOPIC') +OPENMETRICS_FILE = os.environ.get('WIS2_GDC_OPENMETRICS_FILE') PUBLISH_REPORTS = str2bool(os.environ.get('WIS2_GDC_PUBLISH_REPORTS', 'false')) GB_LINKS = [] if None in [API_URL, API_URL_DOCKER, BACKEND_TYPE, - BACKEND_CONNECTION, BROKER_URL, CENTRE_ID, PUBLISH_REPORTS]: + BACKEND_CONNECTION, BROKER_URL, CENTRE_ID, + GB, GB_TOPIC, OPENMETRICS_FILE, PUBLISH_REPORTS]: raise EnvironmentError('Environment variables not set!') for key, value in os.environ.items(): diff --git a/wis2_gdc/registrar.py b/wis2_gdc/registrar.py index 5249189..c9d69f2 100644 --- a/wis2_gdc/registrar.py +++ b/wis2_gdc/registrar.py @@ -33,7 +33,8 @@ from wis2_gdc.backend import BACKENDS from wis2_gdc.env import (BACKEND_TYPE, BACKEND_CONNECTION, BROKER_URL, - GB_LINKS, PUBLISH_REPORTS) + CENTRE_ID, GB_LINKS, PUBLISH_REPORTS) +from wis2_gdc.monitor.metrics import Metrics LOGGER = logging.getLogger(__name__) @@ -48,6 +49,7 @@ def __init__(self): self.broker = None self.metadata = None + self.metrics = Metrics() if PUBLISH_REPORTS: self.broker = MQTTPubSubClient(BROKER_URL) @@ -64,6 +66,7 @@ def register(self, metadata: dict) -> None: self.metadata = metadata self.centre_id = self.metadata['id'].split(':')[3] topic = f"report/a/wis2/{self.centre_id}" + centre_id_labels = [self.centre_id, CENTRE_ID] LOGGER.debug(f'Metadata: {self.metadata}') @@ -79,9 +82,19 @@ def register(self, metadata: dict) -> None: LOGGER.warning('ETS errors; metadata not published') return except KeyError: # validation error - pass - # LOGGER.debug('Validation errors; metadata not published') - # return + self.metrics.failed_total.labels(*centre_id_labels).inc() + # pass + LOGGER.debug('Validation errors; metadata not published') +# self.metrics.write() +# return + + self.metrics.passed_total.labels(*centre_id_labels).inc() + + data_policy = self.metadata['properties']['wmo:dataPolicy'] + if data_policy == 'core': + self.metrics.core_total.labels(*centre_id_labels).inc() + elif data_policy == 'recommended': + self.metrics.recommended_total.labels(*centre_id_labels).inc() LOGGER.info('Updating links') self.update_record_links() @@ -96,6 +109,8 @@ def register(self, metadata: dict) -> None: LOGGER.info('Publishing KPI report to broker') self.broker.pub(topic, json.dumps(kpi_results)) + self.metrics.write() + def _run_ets(self) -> dict: """ Helper function to run ETS