From fefdd35240e25cdbd1507259af1895044a78c868 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 9 Dec 2024 16:04:00 +0100 Subject: [PATCH 1/3] add documentation and volume for custom cvs2bufr-mapping-templates --- docker-compose.yml | 8 +++++++- .../running/data-pipeline-plugins.rst | 18 +++++++++++++++++- docs/source/user/data-ingest.rst | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b971e83f8..fc3e578c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,8 @@ services: wis2box-api: container_name: wis2box-api - image: ghcr.io/wmo-im/wis2box-api:latest + image: wis2box-api:issue-806 + #image: ghcr.io/wmo-im/wis2box-api:latest restart: always env_file: - wis2box.env @@ -40,6 +41,8 @@ services: test: ["CMD", "curl", "-f", "http://localhost/oapi/admin/resources"] interval: 5s retries: 100 + volumes: + - ${WIS2BOX_HOST_DATADIR}/mappings:/data/wis2box/mappings:ro minio: container_name: wis2box-minio @@ -100,6 +103,8 @@ services: context: ./wis2box-broker env_file: - wis2box.env + volumes: + - mosquitto-config:/mosquitto/config wis2box-management: container_name: wis2box-management @@ -150,3 +155,4 @@ volumes: minio-data: auth-data: htpasswd: + mosquitto-config: diff --git a/docs/source/reference/running/data-pipeline-plugins.rst b/docs/source/reference/running/data-pipeline-plugins.rst index 00f2e849e..08d604aca 100644 --- a/docs/source/reference/running/data-pipeline-plugins.rst +++ b/docs/source/reference/running/data-pipeline-plugins.rst @@ -32,7 +32,21 @@ A typical csv2bufr plugin workflow definition would by defined as follows: csv: - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR - template: /data/wis2box/synop_bufr.json # locally created csv2bufr mapping (located in $WIS2BOX_HOST_DATADIR) + template: aws-template # using one of the built-in templates + notify: true # trigger GeoJSON publishing for API and UI + file-pattern: '^.*\.csv$' + +The default templates are defined by the `csv2bufr-templates`_ repository. + +In the case the user wants to use a custom template, the template should be located in the ``$WIS2BOX_HOST_DATADIR/mappings`` directory. + +The plugin configuration would then be defined as follows: + +.. code-block:: yaml + + csv: + - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR + template: /data/wis2box/mappings/my_own_template.json # locally created csv2bufr mapping (located in $WIS2BOX_HOST_DATADIR/mappings) notify: true # trigger GeoJSON publishing for API and UI file-pattern: '^.*\.csv$' @@ -146,5 +160,7 @@ For example, to publish GRIB2 data matching the file-pattern ``^.*_(\d{8})\d{2}. See :ref:`data-mappings` for a full example data mapping configuration. .. _`csv2bufr`: https://csv2bufr.readthedocs.io +.. _`csv2bufr-templates`: https://github.com/wmo-im/csv2bufr-templates .. _`bufr2geojson`: https://github.com/wmo-im/bufr2geojson .. _`synop2bufr`: https://synop2bufr.readthedocs.io + diff --git a/docs/source/user/data-ingest.rst b/docs/source/user/data-ingest.rst index 90705e8f4..2a9d40622 100644 --- a/docs/source/user/data-ingest.rst +++ b/docs/source/user/data-ingest.rst @@ -40,7 +40,7 @@ The wis2box provides 3 types of built-in plugins to publish data in BUFR format: * `bufr2bufr` : the input is received in BUFR format and split by subset, where each subset is published as a separate bufr message * `synop2bufr` : the input is received in `FM-12 SYNOP format `_ and converted to BUFR format. The year and month are extracted from the file pattern -* `csv2bufr` : the input is received in csv format and converted to BUFR format +* `csv2bufr` : the input is received in csv format and converted to BUFR format, a mapping template is used to convert the csv columns to BUFR encoded values. Custom mapping templates need to be placed in the ``$WIS2BOX_HOST_DATADIR/mappings`` directory. See :ref:`csv2bufr-templates` for examples of mapping templates. To publish data for other data formats you can use the 'Universal' plugin, which will pass through the data without any conversion. Please note that you will need to ensure that the date timestamp can be extracted from the file pattern when using this plugin. From 81a3ab6153b10a18ee09486e2d702a93c3820e84 Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 10 Dec 2024 13:38:02 +0100 Subject: [PATCH 2/3] remove accidental commit of local image used --- docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fc3e578c0..51ff15c1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,8 +29,7 @@ services: wis2box-api: container_name: wis2box-api - image: wis2box-api:issue-806 - #image: ghcr.io/wmo-im/wis2box-api:latest + image: ghcr.io/wmo-im/wis2box-api:latest restart: always env_file: - wis2box.env From 9b812f30f8580aab309ce28409534e82aece56d7 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 16 Dec 2024 14:09:25 +0100 Subject: [PATCH 3/3] add mosquitto changes to this branch for later --- wis2box-broker/Dockerfile | 2 - wis2box-broker/acl.conf | 8 ---- wis2box-broker/entrypoint.sh | 69 +++++++++++++++++++++++-------- wis2box-broker/mosquitto-ssl.conf | 24 ----------- wis2box-broker/mosquitto.conf | 1 - 5 files changed, 52 insertions(+), 52 deletions(-) delete mode 100644 wis2box-broker/acl.conf delete mode 100644 wis2box-broker/mosquitto-ssl.conf diff --git a/wis2box-broker/Dockerfile b/wis2box-broker/Dockerfile index 5357feb3e..81dd684ae 100644 --- a/wis2box-broker/Dockerfile +++ b/wis2box-broker/Dockerfile @@ -24,10 +24,8 @@ FROM eclipse-mosquitto:2.0.20 RUN mkdir -p /data/wis2box/mosquitto RUN ln -s /mosquitto /data/wis2box/mosquitto -COPY mosquitto-ssl.conf /mosquitto/config/mosquitto-ssl.conf COPY mosquitto.conf /mosquitto/config/mosquitto.conf -COPY acl.conf /mosquitto/config/acl.conf COPY entrypoint.sh /docker-entrypoint.sh RUN chmod +x /docker-entrypoint.sh diff --git a/wis2box-broker/acl.conf b/wis2box-broker/acl.conf deleted file mode 100644 index 64f9e36b1..000000000 --- a/wis2box-broker/acl.conf +++ /dev/null @@ -1,8 +0,0 @@ -user everyone -topic read origin/# - -user _WIS2BOX_BROKER_USERNAME -topic readwrite origin/# -topic readwrite wis2box/# -topic readwrite data-incoming/# -topic read $SYS/# \ No newline at end of file diff --git a/wis2box-broker/entrypoint.sh b/wis2box-broker/entrypoint.sh index f3e0f6193..e6a5192b1 100644 --- a/wis2box-broker/entrypoint.sh +++ b/wis2box-broker/entrypoint.sh @@ -1,28 +1,49 @@ #!/bin/sh -if [ -f /tmp/wis2box.crt ]; then - echo "SSL enabled" - echo "setup /mosquitto/certs" - mkdir -p /mosquitto/certs - cp /tmp/wis2box.crt /mosquitto/certs - cp /tmp/wis2box.key /mosquitto/certs - chown -R mosquitto:mosquitto /mosquitto/certs - cp -f /mosquitto/config/mosquitto-ssl.conf /mosquitto/config/mosquitto.conf -else - echo "SSL disabled" -fi - echo "Setting mosquitto authentication" if [ ! -e "/mosquitto/config/password.txt" ]; then echo "Adding wis2box users to mosquitto password file" mosquitto_passwd -b -c /mosquitto/config/password.txt $WIS2BOX_BROKER_USERNAME $WIS2BOX_BROKER_PASSWORD mosquitto_passwd -b /mosquitto/config/password.txt everyone everyone else - echo "Mosquitto password file already exists. Skipping wis2box user addition." + echo "Mosquitto password file already exists. Update it if needed" + mosquitto_passwd -b /mosquitto/config/password.txt everyone everyone + mosquitto_passwd -b /mosquitto/config/password.txt $WIS2BOX_BROKER_USERNAME $WIS2BOX_BROKER_PASSWORD +fi + +# add max_queued_messages to mosquitto.conf if not already there +if ! grep -q "max_queued_messages" /mosquitto/config/mosquitto.conf; then + echo "max_queued_messages $WIS2BOX_BROKER_QUEUE_MAX" >> /mosquitto/config/mosquitto.conf fi -sed -i "s#_WIS2BOX_BROKER_QUEUE_MAX#$WIS2BOX_BROKER_QUEUE_MAX#" /mosquitto/config/mosquitto.conf -sed -i "s#_WIS2BOX_BROKER_USERNAME#$WIS2BOX_BROKER_USERNAME#" /mosquitto/config/acl.conf +# prepare the acl.conf file +if [ ! -e "/mosquitto/config/acl.conf" ]; then + echo "Creating mosquitto acl file" + echo "user everyone" >> /mosquitto/config/acl.conf + echo "topic read origin/#" >> /mosquitto/config/acl.conf + echo " " >> /mosquitto/config/acl.conf + echo "user $WIS2BOX_BROKER_USERNAME" >> /mosquitto/config/acl.conf + echo "topic readwrite origin/#" >> /mosquitto/config/acl.conf + echo "topic readwrite wis2box/#" >> /mosquitto/config/acl.conf + echo "topic readwrite data-incoming/#" >> /mosquitto/config/acl.conf + echo "topic read \$SYS/#" >> /mosquitto/config/acl.conf +else + echo "Mosquitto acl file already exists. Update it if needed" + # add user everyone to acl.conf if not already there + if ! grep -q "user everyone" /mosquitto/config/acl.conf; then + echo "user everyone" >> /mosquitto/config/acl.conf + echo "topic read origin/#" >> /mosquitto/config/acl.conf + echo " " >> /mosquitto/config/acl.conf + fi + # add user $WIS2BOX_BROKER_USERNAME to acl.conf if not already there + if ! grep -q "user $WIS2BOX_BROKER_USERNAME" /mosquitto/config/acl.conf; then + echo "user $WIS2BOX_BROKER_USERNAME" >> /mosquitto/config/acl.conf + echo "topic readwrite origin/#" >> /mosquitto/config/acl.conf + echo "topic readwrite wis2box/#" >> /mosquitto/config/acl.conf + echo "topic readwrite data-incoming/#" >> /mosquitto/config/acl.conf + echo "topic read \$SYS/#" >> /mosquitto/config/acl.conf + fi +fi for i in `env | grep -Ee "\> /mosquitto/config/acl.conf done -# set ownership of mosquitto files -chown -R mosquitto:mosquitto /mosquitto +if [ -f /tmp/wis2box.crt ]; then + echo "SSL enabled" + echo "setup /mosquitto/certs" + mkdir -p /mosquitto/certs + cp /tmp/wis2box.crt /mosquitto/certs + cp /tmp/wis2box.key /mosquitto/certs + chown -R mosquitto:mosquitto /mosquitto/certs + # add listener 8883 block to mosquitto.conf, if not already there + if ! grep -q "listener 8883" /mosquitto/config/mosquitto.conf; then + echo "listener 8883" >> /mosquitto/config/mosquitto.conf + echo "certfile /mosquitto/certs/wis2box.crt" >> /mosquitto/config/mosquitto.conf + echo "keyfile /mosquitto/certs/wis2box.key" >> /mosquitto/config/mosquitto.conf + fi +else + echo "SSL disabled" +fi # set permission of acl.conf to 0700 chmod 0700 /mosquitto/config/acl.conf diff --git a/wis2box-broker/mosquitto-ssl.conf b/wis2box-broker/mosquitto-ssl.conf deleted file mode 100644 index 44a7fc6ab..000000000 --- a/wis2box-broker/mosquitto-ssl.conf +++ /dev/null @@ -1,24 +0,0 @@ -persistence true -persistence_location /mosquitto/data/ -log_dest file /mosquitto/log/mosquitto.log -log_dest stdout -log_timestamp_format %Y-%m-%dT%H:%M:%S -password_file /mosquitto/config/password.txt -max_queued_messages _WIS2BOX_BROKER_QUEUE_MAX - -# ACLs -acl_file /mosquitto/config/acl.conf - -## MQTT Listener -listener 1883 -protocol mqtt - -## WebSockets Listener -listener 8884 -protocol websockets - -## MQTTs -listener 8883 -certfile /mosquitto/certs/wis2box.crt -keyfile /mosquitto/certs/wis2box.key - diff --git a/wis2box-broker/mosquitto.conf b/wis2box-broker/mosquitto.conf index a00c78440..ed42b5ca3 100644 --- a/wis2box-broker/mosquitto.conf +++ b/wis2box-broker/mosquitto.conf @@ -4,7 +4,6 @@ log_dest file /mosquitto/log/mosquitto.log log_dest stdout log_timestamp_format %Y-%m-%dT%H:%M:%S password_file /mosquitto/config/password.txt -max_queued_messages _WIS2BOX_BROKER_QUEUE_MAX # ACLs acl_file /mosquitto/config/acl.conf