add metrics support

wmo-im · Jan 31, 2024 · 24f7411 · 24f7411
1 parent 6ed971a
commit 24f7411
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 8 deletions.
diff --git a/docker/wis2-gdc.env b/docker/wis2-gdc.env
@@ -6,6 +6,7 @@ export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@mosquitto:1883
 export WIS2_GDC_CENTRE_ID=ca-eccc-msc-gdc
 export WIS2_GDC_GB=mqtt://everyone:everyone@mosquitto:1883
 export WIS2_GDC_GB_TOPIC=origin/a/wis2/+/+/metadata/#
+export WIS2_GDC_OPENMETRICS_FILE=/tmp/wis2-gdc-openmetrics.txt
 export WIS2_GDC_PUBLISH_REPORTS=true
 
 # global broker links

diff --git a/wis2-gdc.env b/wis2-gdc.env
@@ -6,6 +6,7 @@ export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@localhost:1883
 export WIS2_GDC_CENTRE_ID=ca-eccc-msc-gdc
 export WIS2_GDC_GB=mqtt://everyone:everyone@localhost:1883
 export WIS2_GDC_GB_TOPIC=origin/a/wis2/+/+/metadata/#
+export WIS2_GDC_OPENMETRICS_FILE=/tmp/wis2-gdc-openmetrics.txt
 export WIS2_GDC_PUBLISH_REPORTS=true
 
 # global broker links

diff --git a/wis2_gdc/__init__.py b/wis2_gdc/__init__.py
@@ -22,6 +22,7 @@
 import click
 
 from wis2_gdc.registrar import register, setup, teardown
+from wis2_gdc.monitor import monitor
 from wis2_gdc.sync import archive, sync
 
 __version__ = '0.1.dev0'
@@ -35,8 +36,9 @@ def cli():
     pass
 
 
-cli.add_command(archive)
-cli.add_command(sync)
-cli.add_command(register)
 cli.add_command(setup)
 cli.add_command(teardown)
+cli.add_command(register)
+cli.add_command(sync)
+cli.add_command(monitor)
+cli.add_command(archive)
diff --git a/wis2_gdc/env.py b/wis2_gdc/env.py
@@ -49,12 +49,16 @@ def str2bool(value: Any) -> bool:
 BACKEND_CONNECTION = os.environ.get('WIS2_GDC_BACKEND_CONNECTION')
 BROKER_URL = os.environ.get('WIS2_GDC_BROKER_URL')
 CENTRE_ID = os.environ.get('WIS2_GDC_CENTRE_ID')
+GB = os.environ.get('WIS2_GDC_GB')
+GB_TOPIC = os.environ.get('WIS2_GDC_GB_TOPIC')
+OPENMETRICS_FILE = os.environ.get('WIS2_GDC_OPENMETRICS_FILE')
 PUBLISH_REPORTS = str2bool(os.environ.get('WIS2_GDC_PUBLISH_REPORTS', 'false'))
 
 GB_LINKS = []
 
 if None in [API_URL, API_URL_DOCKER, BACKEND_TYPE,
-            BACKEND_CONNECTION, BROKER_URL, CENTRE_ID, PUBLISH_REPORTS]:
+            BACKEND_CONNECTION, BROKER_URL, CENTRE_ID,
+            GB, GB_TOPIC, OPENMETRICS_FILE, PUBLISH_REPORTS]:
     raise EnvironmentError('Environment variables not set!')
 
 for key, value in os.environ.items():

diff --git a/wis2_gdc/registrar.py b/wis2_gdc/registrar.py
@@ -33,7 +33,8 @@
 
 from wis2_gdc.backend import BACKENDS
 from wis2_gdc.env import (BACKEND_TYPE, BACKEND_CONNECTION, BROKER_URL,
-                          GB_LINKS, PUBLISH_REPORTS)
+                          CENTRE_ID, GB_LINKS, PUBLISH_REPORTS)
+from wis2_gdc.monitor.metrics import Metrics
 
 LOGGER = logging.getLogger(__name__)
 
@@ -48,6 +49,7 @@ def __init__(self):
 
         self.broker = None
         self.metadata = None
+        self.metrics = Metrics()
 
         if PUBLISH_REPORTS:
             self.broker = MQTTPubSubClient(BROKER_URL)
@@ -64,6 +66,7 @@ def register(self, metadata: dict) -> None:
         self.metadata = metadata
         self.centre_id = self.metadata['id'].split(':')[3]
         topic = f"report/a/wis2/{self.centre_id}"
+        centre_id_labels = [self.centre_id, CENTRE_ID]
 
         LOGGER.debug(f'Metadata: {self.metadata}')
 
@@ -79,9 +82,19 @@ def register(self, metadata: dict) -> None:
                 LOGGER.warning('ETS errors; metadata not published')
                 return
         except KeyError:  # validation error
-            pass
-            # LOGGER.debug('Validation errors; metadata not published')
-            # return
+            self.metrics.failed_total.labels(*centre_id_labels).inc()
+            # pass
+            LOGGER.debug('Validation errors; metadata not published')
+#            self.metrics.write()
+#            return
+
+        self.metrics.passed_total.labels(*centre_id_labels).inc()
+
+        data_policy = self.metadata['properties']['wmo:dataPolicy']
+        if data_policy == 'core':
+            self.metrics.core_total.labels(*centre_id_labels).inc()
+        elif data_policy == 'recommended':
+            self.metrics.recommended_total.labels(*centre_id_labels).inc()
 
         LOGGER.info('Updating links')
         self.update_record_links()
@@ -96,6 +109,8 @@ def register(self, metadata: dict) -> None:
             LOGGER.info('Publishing KPI report to broker')
             self.broker.pub(topic, json.dumps(kpi_results))
 
+        self.metrics.write()
+
     def _run_ets(self) -> dict:
         """
         Helper function to run ETS