diff --git a/Makefile b/Makefile index 50df219..2e4636f 100644 --- a/Makefile +++ b/Makefile @@ -1,51 +1,55 @@ -# ================================================================= +############################################################################### # -# Authors: Tom Kralidis +# Copyright (C) 2025 Tom Kralidis # -# Copyright (c) 2024 Tom Kralidis +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= +############################################################################### + +DOCKER_COMPOSE_ARGS=--project-name msc-wis2node --file docker-compose.yml --file docker-compose.override.yml + +build: + sudo docker compose $(DOCKER_COMPOSE_ARGS) build + +force-build: + sudo docker compose $(DOCKER_COMPOSE_ARGS) build --no-cache -# Ubuntu -SR3_CONFIG=${HOME}/.config/sr3 +up: + sudo docker compose $(DOCKER_COMPOSE_ARGS) up -# Mac OSX -#SR3_CONFIG=${HOME}/Library/Application\ Support/sr3 +down: + sudo docker compose $(DOCKER_COMPOSE_ARGS) down -check: - @echo "SR3 configuration directory: ${SR3_CONFIG}" +restart: down up -install: setup - cp msc_wis2node/publisher.py $(SR3_CONFIG)/plugins - cp deploy/default/sarracenia/dd.weather.gc.ca-all.conf $(SR3_CONFIG)/subscribe +login: + docker exec -it msc-wis2node-management /bin/bash -setup: - mkdir -p $(SR3_CONFIG)/plugins - mkdir -p $(SR3_CONFIG)/subscribe +dev: + sudo docker compose $(DOCKER_COMPOSE_ARGS) --file docker-compose.dev.yml up + +reinit-backend: + docker exec -it msc-wis2node-management sh -c "msc-wis2node setup --force" + +logs: + sudo docker compose $(DOCKER_COMPOSE_ARGS) logs --follow clean: - rm -fr $(SR3_CONFIG)/plugins/publisher.py - rm -fr $(SR3_CONFIG)/subscribe/dd.weather.gc.ca-all.conf + docker system prune -f + docker volume prune -f + +rm: + docker volume rm $(shell docker volume ls --filter name=msc-wis2node -q) -.PHONY: check install setup clean +.PHONY: build up dev login down restart reinit-backend force-build logs rm clean diff --git a/README.md b/README.md index acd184c..4997c77 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ source bin/activate # clone codebase and install git clone https://github.com/ECCC-MSC/msc-wis2node.git -cd msc-wis2node +cd msc-wis2node/msc-wis2node-management python3 setup.py install # install sarracenia configurations @@ -55,9 +55,9 @@ vim local.env # update accordingly # MSC_WIS2NODE_BROKER_USERNAME: username of the MQTT broker to publish to=admin # MSC_WIS2NODE_BROKER_PASSWORD: password of the MQTT broker to publish to # MSC_WIS2NODE_MSC_DATAMART_AMQP: URL to MSC Datamart notification service -# MSC_WIS2NODE_DATASET_CONFIG: filepath where MSC dataset definitions are managed # MSC_WIS2NODE_DISCOVERY_METADATA_ZIP_URL: URL to SSC GitLab zipfile of MSC discovery metadata # MSC_WIS2NODE_TOPIC_PREFIX: base topic prefix for publication (i.e. origin/a/wis2/ca-eccc-msc) +# MSC_WIS2NODE_CACHE: optional memcache instance source local.env @@ -80,7 +80,46 @@ msc-wis2node dataset delete-metadata --metadata-id 12345 ### Docker -Instructions to run msc-wis2node via Docker can be found in the [`docker`](docker) directory. +The Docker setup uses Docker and Docker Compose to manage the following services: + +- **msc-wis2node-cache**: memcache caching for data update detection (optional) +- **msc-wis2node-management**: management service to subscribe to MSC Datamart/HPFX and re-publish to WIS2 + +See [`msc-wis2node.env`](msc-wis2node.env) for default environment variable settings. + +To adjust service ports, edit [`docker-compose.override.yml`](docker-compose.override.yml) accordingly. + +The [`Makefile`](Makefile) in the root directory provides options to manage the Docker Compose setup. + +```bash +# build all images +make build + +# build all images (no cache) +make force-build + +# start all containers +make up + +# start all containers in dev mode +make dev + +# view all container logs in realtime +make logs + +# login to the msc-wis2node-management container +make login + +# restart all containers +make restart + +# shutdown all containers +make down + +# remove all volumes +make rm +``` + ## Development diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 8e8e5be..0000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -msc-wis2node (0.1.0) focal; urgency=medium - - * initial Debian packaging - - -- Tom Kralidis Tue, 16 Jan 2024 17:24:50 +0000 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index 45a4fb7..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -8 diff --git a/debian/control b/debian/control deleted file mode 100644 index aa5daec..0000000 --- a/debian/control +++ /dev/null @@ -1,14 +0,0 @@ -Source: msc-wis2node -Section: python -Priority: optional -Maintainer: Tom Kralidis -Build-Depends: debhelper (>= 9), python3, python3-setuptools -Standards-Version: 3.9.5 -X-Python-Version: >= 3.6 -Vcs-Git: https://github.com/ECCC-MSC/msc-wis2node.git - -Package: msc-wis2node -Architecture: all -Depends: python3, python3-click, python3-metpx-sarracenia, pywis-pubsub -Homepage: https://github.com/ECCC-MSC/msc-wis2node -Description: MSC WMO WIS2 Node implementation diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 0058a03..0000000 --- a/debian/copyright +++ /dev/null @@ -1,24 +0,0 @@ -Upstream: msc-wis2node -Source: https://github.com/ECCC-MSC/msc-wis2node -Files: * -Copyright: 2024 Government of Canada -License: GPL-3+ - MSC WMO WIS2 Node implementation - Copyright (C) 2024 Government of Canada - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - - On Debian systems, the full text of the GNU General Public - License version 2 can be found in the file - `/usr/share/common-licenses/GPL-3'. diff --git a/debian/msc-wis2node.install b/debian/msc-wis2node.install deleted file mode 100644 index e314696..0000000 --- a/debian/msc-wis2node.install +++ /dev/null @@ -1 +0,0 @@ -deploy/default/sarracenia/*.conf opt/msc-wis2node/etc/sarracenia diff --git a/debian/rules b/debian/rules deleted file mode 100644 index 9fd7abe..0000000 --- a/debian/rules +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile -*- -# Sample debian/rules that uses debhelper. -# This file was originally written by Joey Hess and Craig Small. -# As a special exception, when this file is copied by dh-make into a -# dh-make output file, you may use that output file without restriction. -# This special exception was added by Craig Small in version 0.37 of dh-make. - -# Uncomment this to turn on verbose mode. -#export DH_VERBOSE=1 -#export DH_OPTIONS=-v - -%: - dh $@ --with python3 --buildsystem=pybuild diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 163aaf8..0000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (quilt) diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..58cc415 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,23 @@ +############################################################################### +# +# Copyright (C) 2025 Tom Kralidis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +services: + msc-wis2node-cache: + ports: + - 11211:11211 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2bdf803 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +############################################################################### +# +# Copyright (C) 2025 Tom Kralidis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +services: + msc-wis2node-cache: + image: memcached:latest + container_name: msc-wis2node-cache + env_file: + - msc-wis2node.env + restart: unless-stopped + msc-wis2node-management: + image: msc-wis2node-management + container_name: msc-wis2node + build: + context: msc-wis2node-management/ + env_file: + - msc-wis2node.env + restart: unless-stopped diff --git a/msc-wis2node-management/Dockerfile b/msc-wis2node-management/Dockerfile new file mode 100644 index 0000000..4a07d8f --- /dev/null +++ b/msc-wis2node-management/Dockerfile @@ -0,0 +1,59 @@ +############################################################################### +# +# Copyright (C) 2025 Tom Kralidis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +FROM ubuntu:jammy + +LABEL maintainer="tomkralidis@gmail.com" + +ARG SR3_CONFIG /home/msc-wis2node/.config/sr3 + +ENV TZ="Etc/UTC" \ + DEBIAN_FRONTEND="noninteractive" \ + DEBIAN_PACKAGES="bash curl git python3-pip python3-setuptools sudo vim" \ + MSC_WIS2NODE_DATASET_CONFIG=/home/msc-wis2node/datasets.yml + +# copy the app +COPY ./ /app + +RUN apt-get update -y && \ + # install dependencies + apt-get install -y ${DEBIAN_PACKAGES} && \ + pip3 install --no-cache-dir -r /app/requirements.txt && \ + # install msc-wis2node + cd /app && \ + pip3 install -e . && \ + chmod +x /app/docker/entrypoint.sh && \ + # cleanup + apt autoremove -y && \ + apt-get -q clean && \ + rm -rf /var/lib/apt/lists/* && \ + # add msc-wis2node user + useradd -ms /bin/bash msc-wis2node && \ + adduser msc-wis2node sudo && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \ + # install sr3 components + mkdir -p /home/msc-wis2node/.config/sr3/plugins && \ + mkdir -p /home/msc-wis2node/.config/sr3/subscribe && \ + cp msc_wis2node/publisher.py /home/msc-wis2node/.config/sr3/plugins/publisher.py && \ + cp deploy/default/sarracenia/dd.weather.gc.ca-all.conf /home/msc-wis2node/.config/sr3/subscribe/dd.weather.gc.ca-all.conf + +USER msc-wis2node +WORKDIR /home/msc-wis2node + +ENTRYPOINT [ "/app/docker/entrypoint.sh" ] diff --git a/msc-wis2node-management/MANIFEST.in b/msc-wis2node-management/MANIFEST.in new file mode 100644 index 0000000..b2cc4f5 --- /dev/null +++ b/msc-wis2node-management/MANIFEST.in @@ -0,0 +1 @@ +include README.md LICENSE requirements.txt diff --git a/msc-wis2node-management/README.md b/msc-wis2node-management/README.md new file mode 100644 index 0000000..f76fe76 --- /dev/null +++ b/msc-wis2node-management/README.md @@ -0,0 +1,3 @@ +# msc-wis2node + +Python package to perform MSC WIS2 Node management functions. diff --git a/deploy/default/sarracenia/dd.weather.gc.ca-all.conf b/msc-wis2node-management/deploy/default/sarracenia/dd.weather.gc.ca-all.conf similarity index 100% rename from deploy/default/sarracenia/dd.weather.gc.ca-all.conf rename to msc-wis2node-management/deploy/default/sarracenia/dd.weather.gc.ca-all.conf diff --git a/msc-wis2node-management/docker/entrypoint.sh b/msc-wis2node-management/docker/entrypoint.sh new file mode 100755 index 0000000..33a09d0 --- /dev/null +++ b/msc-wis2node-management/docker/entrypoint.sh @@ -0,0 +1,33 @@ +#!/bin/bash +############################################################################### +# +# Copyright (C) 2025 Tom Kralidis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +echo "START /entrypoint.sh" + +printenv | grep -v "no_proxy" > /tmp/environment +sudo sh -c 'cat /tmp/environment >> /etc/environment' +rm -f /tmp/environment + +echo "Setting up MSC dataset config" +msc-wis2node dataset setup + +echo "starting sr3..." +sr3 --logStdout start subscribe/dd.weather.gc.ca-all && sleep infinity + +echo "END /entrypoint.sh" diff --git a/msc_wis2node/__init__.py b/msc-wis2node-management/msc_wis2node/__init__.py similarity index 100% rename from msc_wis2node/__init__.py rename to msc-wis2node-management/msc_wis2node/__init__.py diff --git a/msc_wis2node/cli_options.py b/msc-wis2node-management/msc_wis2node/cli_options.py similarity index 100% rename from msc_wis2node/cli_options.py rename to msc-wis2node-management/msc_wis2node/cli_options.py diff --git a/msc_wis2node/dataset.py b/msc-wis2node-management/msc_wis2node/dataset.py similarity index 100% rename from msc_wis2node/dataset.py rename to msc-wis2node-management/msc_wis2node/dataset.py diff --git a/msc_wis2node/env.py b/msc-wis2node-management/msc_wis2node/env.py similarity index 91% rename from msc_wis2node/env.py rename to msc-wis2node-management/msc_wis2node/env.py index deec57f..23472ab 100644 --- a/msc_wis2node/env.py +++ b/msc-wis2node-management/msc_wis2node/env.py @@ -1,6 +1,6 @@ ############################################################################### # -# Copyright (C) 2024 Tom Kralidis +# Copyright (C) 2025 Tom Kralidis # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,6 +31,8 @@ DATASET_CONFIG = os.environ.get('MSC_WIS2NODE_DATASET_CONFIG') TOPIC_PREFIX = os.environ.get('MSC_WIS2NODE_TOPIC_PREFIX', 'origin/a/wis2') DISCOVERY_METADATA_ZIP_URL = os.environ.get('MSC_WIS2NODE_DISCOVERY_METADATA_ZIP_URL') # noqa +CACHE = os.environ.get('MSC_WIS2NODE_CACHE') +CACHE_EXPIRY_SECONDS = int(os.environ.get('MSC_WIS2NODE_CACHE_EXPIRY_SECONDS', 86400)) # noqa CENTRE_ID = os.environ.get('MSC_WIS2NODE_CENTRE_ID') WIS2_GDC = os.environ.get('MSC_WIS2NODE_WIS2_GDC') diff --git a/msc_wis2node/publisher.py b/msc-wis2node-management/msc_wis2node/publisher.py similarity index 90% rename from msc_wis2node/publisher.py rename to msc-wis2node-management/msc_wis2node/publisher.py index c69e330..e648e5f 100644 --- a/msc_wis2node/publisher.py +++ b/msc-wis2node-management/msc_wis2node/publisher.py @@ -17,6 +17,7 @@ # ############################################################################### +from copy import deepcopy from datetime import date, datetime, timezone from fnmatch import fnmatch import json @@ -26,13 +27,14 @@ import uuid from paho.mqtt import publish +from pymemcache.client import base as memcache_base from pywis_pubsub.publish import create_message from sarracenia.flowcb import FlowCB import yaml from msc_wis2node.env import (BROKER_HOSTNAME, BROKER_PORT, BROKER_USERNAME, - BROKER_PASSWORD, CENTRE_ID, DATASET_CONFIG, - TOPIC_PREFIX) + BROKER_PASSWORD, CACHE, CACHE_EXPIRY_SECONDS, + CENTRE_ID, DATASET_CONFIG, TOPIC_PREFIX) from msc_wis2node.util import get_mqtt_client_id, get_mqtt_tls_settings LOGGER = logging.getLogger(__name__) @@ -74,11 +76,15 @@ class WIS2Publisher: def __init__(self): """initialize""" + self.cache = None self.datasets = [] self.tls = None self.client_id = get_mqtt_client_id() + if CACHE is not None: + self.cache = memcache_base.Client(CACHE) + if BROKER_PORT == 8883: self.tls = get_mqtt_tls_settings() @@ -178,6 +184,16 @@ def publish_to_wis2(self, dataset: dict, url: str) -> None: url=url ) + if self.cache is not None: + LOGGER.debug('Checking for data update') + if self.cache.get(message['properties']['data_id']) is not None: + update_link = deepcopy(message['links'][0]) + update_link['rel'] = 'update' + message['links'].append(update_link) + else: + self.cache.set(message['properties']['data_id'], + CACHE_EXPIRY_SECONDS) + cache = dataset.get('cache', True) if not cache: LOGGER.info(f'Setting properties.cache={cache}') diff --git a/msc_wis2node/util.py b/msc-wis2node-management/msc_wis2node/util.py similarity index 100% rename from msc_wis2node/util.py rename to msc-wis2node-management/msc_wis2node/util.py diff --git a/requirements-dev.txt b/msc-wis2node-management/requirements-dev.txt similarity index 100% rename from requirements-dev.txt rename to msc-wis2node-management/requirements-dev.txt diff --git a/requirements.txt b/msc-wis2node-management/requirements.txt similarity index 79% rename from requirements.txt rename to msc-wis2node-management/requirements.txt index 0d7975d..8d412d8 100644 --- a/requirements.txt +++ b/msc-wis2node-management/requirements.txt @@ -3,3 +3,4 @@ certifi click metpx-sr3 pywis-pubsub +pymemcache diff --git a/setup.py b/msc-wis2node-management/setup.py similarity index 100% rename from setup.py rename to msc-wis2node-management/setup.py diff --git a/msc-wis2node.env b/msc-wis2node.env index e750e3d..a4bebfa 100644 --- a/msc-wis2node.env +++ b/msc-wis2node.env @@ -3,7 +3,9 @@ export MSC_WIS2NODE_BROKER_PORT=8883 export MSC_WIS2NODE_BROKER_USERNAME=username export MSC_WIS2NODE_BROKER_PASSWORD=password export MSC_WIS2NODE_MSC_DATAMART_AMQP=amqps://dd.weather.gc.ca -export MSC_WIS2NODE_DATASET_CONFIG=/opt/msc-wis2node/conf/datasets.yaml export MSC_WIS2NODE_DISCOVERY_METADATA_ZIP_URL=https://example.org/discovery-metadata.zip +export MSC_WIS2NODE_CACHE=msc-wis2node-cache:11211 +export MSC_WIS2NODE_CACHE_EXPIRY_SECONDS=86400 export MSC_WIS2NODE_CENTRE_ID=ca-eccc-msc export MSC_WIS2NODE_TOPIC_PREFIX=origin/a/wis2 +export MSC_WIS2NODE_WIS2_GDC=https://wis2-gdc.weather.gc.ca/collections/wis2-discovery-metadata