Docker setup (#4)

* Dockerize

* fix flake8

* update

* update entrypoint and dataset setup, make cache optional

* update instructions add auto-restart to Docker Compose

* remove debian setup

Makefile

@@ -1,51 +1,55 @@
-# =================================================================
+###############################################################################
 #
-# Authors: Tom Kralidis <[email protected]>
+# Copyright (C) 2025 Tom Kralidis
 #
-# Copyright (c) 2024 Tom Kralidis
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
 #
-# Permission is hereby granted, free of charge, to any person
-# obtaining a copy of this software and associated documentation
-# files (the "Software"), to deal in the Software without
-# restriction, including without limitation the rights to use,
-# copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following
-# conditions:
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
+# GNU General Public License for more details.
 #
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-# =================================================================
+###############################################################################
+
+DOCKER_COMPOSE_ARGS=--project-name msc-wis2node --file docker-compose.yml --file docker-compose.override.yml
+
+build:
+	sudo docker compose $(DOCKER_COMPOSE_ARGS) build
+
+force-build:
+	sudo docker compose $(DOCKER_COMPOSE_ARGS) build --no-cache
 
-# Ubuntu
-SR3_CONFIG=${HOME}/.config/sr3
+up:
+	sudo docker compose $(DOCKER_COMPOSE_ARGS) up
 
-# Mac OSX
-#SR3_CONFIG=${HOME}/Library/Application\ Support/sr3
+down:
+	sudo docker compose $(DOCKER_COMPOSE_ARGS) down
 
-check:
-	@echo "SR3 configuration directory: ${SR3_CONFIG}"
+restart: down up
 
-install: setup
-	cp msc_wis2node/publisher.py $(SR3_CONFIG)/plugins
-	cp deploy/default/sarracenia/dd.weather.gc.ca-all.conf $(SR3_CONFIG)/subscribe
+login:
+	docker exec -it msc-wis2node-management /bin/bash
 
-setup:
-	mkdir -p $(SR3_CONFIG)/plugins
-	mkdir -p $(SR3_CONFIG)/subscribe
+dev:
+	sudo docker compose $(DOCKER_COMPOSE_ARGS) --file docker-compose.dev.yml up
+
+reinit-backend:
+	docker exec -it msc-wis2node-management sh -c "msc-wis2node setup --force"
+
+logs:
+	sudo docker compose $(DOCKER_COMPOSE_ARGS) logs --follow
 
 clean:
-	rm -fr $(SR3_CONFIG)/plugins/publisher.py
-	rm -fr $(SR3_CONFIG)/subscribe/dd.weather.gc.ca-all.conf
+	docker system prune -f
+	docker volume prune -f
+
+rm:
+	docker volume rm $(shell docker volume ls --filter name=msc-wis2node -q)
 
-.PHONY: check install setup clean
+.PHONY: build up dev login down restart reinit-backend force-build logs rm clean

README.md

@@ -28,7 +28,7 @@ source bin/activate
 
 # clone codebase and install
 git clone https://github.com/ECCC-MSC/msc-wis2node.git
-cd msc-wis2node
+cd msc-wis2node/msc-wis2node-management
 python3 setup.py install
 
 # install sarracenia configurations
@@ -55,9 +55,9 @@ vim local.env  # update accordingly
 # MSC_WIS2NODE_BROKER_USERNAME: username of the MQTT broker to publish to=admin
 # MSC_WIS2NODE_BROKER_PASSWORD: password of the MQTT broker to publish to
 # MSC_WIS2NODE_MSC_DATAMART_AMQP: URL to MSC Datamart notification service
-# MSC_WIS2NODE_DATASET_CONFIG: filepath where MSC dataset definitions are managed
 # MSC_WIS2NODE_DISCOVERY_METADATA_ZIP_URL: URL to SSC GitLab zipfile of MSC discovery metadata
 # MSC_WIS2NODE_TOPIC_PREFIX: base topic prefix for publication (i.e. origin/a/wis2/ca-eccc-msc)
+# MSC_WIS2NODE_CACHE: optional memcache instance
 
 source local.env
 
@@ -80,7 +80,46 @@ msc-wis2node dataset delete-metadata --metadata-id 12345
 
 ### Docker
 
-Instructions to run msc-wis2node via Docker can be found in the [`docker`](docker) directory.
+The Docker setup uses Docker and Docker Compose to manage the following services:
+
+- **msc-wis2node-cache**: memcache caching for data update detection (optional)
+- **msc-wis2node-management**: management service to subscribe to MSC Datamart/HPFX and re-publish to WIS2
+
+See [`msc-wis2node.env`](msc-wis2node.env) for default environment variable settings.
+
+To adjust service ports, edit [`docker-compose.override.yml`](docker-compose.override.yml) accordingly.
+
+The [`Makefile`](Makefile) in the root directory provides options to manage the Docker Compose setup.
+
+```bash
+# build all images
+make build
+
+# build all images (no cache)
+make force-build
+
+# start all containers
+make up
+
+# start all containers in dev mode
+make dev
+
+# view all container logs in realtime
+make logs
+
+# login to the msc-wis2node-management container
+make login
+
+# restart all containers
+make restart
+
+# shutdown all containers
+make down
+
+# remove all volumes
+make rm
+```
+
 
 ## Development
 

docker-compose.override.yml

@@ -0,0 +1,23 @@
+###############################################################################
+#
+# Copyright (C) 2025 Tom Kralidis
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+services:
+  msc-wis2node-cache:
+    ports:
+        - 11211:11211

docker-compose.yml

@@ -0,0 +1,34 @@
+###############################################################################
+#
+# Copyright (C) 2025 Tom Kralidis
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+services:
+  msc-wis2node-cache:
+    image: memcached:latest
+    container_name: msc-wis2node-cache
+    env_file:
+      - msc-wis2node.env
+    restart: unless-stopped
+  msc-wis2node-management:
+    image: msc-wis2node-management
+    container_name: msc-wis2node
+    build:
+      context: msc-wis2node-management/
+    env_file:
+      - msc-wis2node.env
+    restart: unless-stopped

msc-wis2node-management/Dockerfile

@@ -0,0 +1,59 @@
+###############################################################################
+#
+# Copyright (C) 2025 Tom Kralidis
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+FROM ubuntu:jammy
+
+LABEL maintainer="[email protected]"
+
+ARG SR3_CONFIG /home/msc-wis2node/.config/sr3
+
+ENV TZ="Etc/UTC" \
+    DEBIAN_FRONTEND="noninteractive" \
+    DEBIAN_PACKAGES="bash curl git python3-pip python3-setuptools sudo vim" \
+    MSC_WIS2NODE_DATASET_CONFIG=/home/msc-wis2node/datasets.yml
+
+# copy the app 
+COPY ./ /app
+
+RUN apt-get update -y && \
+    # install dependencies
+    apt-get install -y ${DEBIAN_PACKAGES} && \
+    pip3 install --no-cache-dir -r /app/requirements.txt && \
+    # install msc-wis2node
+    cd /app && \
+    pip3 install -e . && \
+    chmod +x /app/docker/entrypoint.sh && \
+    # cleanup
+    apt autoremove -y && \
+    apt-get -q clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    # add msc-wis2node user
+    useradd -ms /bin/bash msc-wis2node && \
+    adduser msc-wis2node sudo && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    # install sr3 components
+    mkdir -p /home/msc-wis2node/.config/sr3/plugins && \
+    mkdir -p /home/msc-wis2node/.config/sr3/subscribe && \
+    cp msc_wis2node/publisher.py /home/msc-wis2node/.config/sr3/plugins/publisher.py && \
+    cp deploy/default/sarracenia/dd.weather.gc.ca-all.conf /home/msc-wis2node/.config/sr3/subscribe/dd.weather.gc.ca-all.conf
+
+USER msc-wis2node
+WORKDIR /home/msc-wis2node
+
+ENTRYPOINT [ "/app/docker/entrypoint.sh" ]

msc-wis2node-management/MANIFEST.in

@@ -0,0 +1 @@
+include README.md LICENSE requirements.txt

msc-wis2node-management/README.md

@@ -0,0 +1,3 @@
+# msc-wis2node
+
+Python package to perform MSC WIS2 Node management functions.

msc-wis2node-management/docker/entrypoint.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+###############################################################################
+#
+# Copyright (C) 2025 Tom Kralidis
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+echo "START /entrypoint.sh"
+
+printenv | grep -v "no_proxy" > /tmp/environment
+sudo sh -c 'cat /tmp/environment >> /etc/environment'
+rm -f /tmp/environment
+
+echo "Setting up MSC dataset config"
+msc-wis2node dataset setup
+
+echo "starting sr3..."
+sr3 --logStdout start subscribe/dd.weather.gc.ca-all && sleep infinity
+
+echo "END /entrypoint.sh"