diff --git a/docker-compose.yml b/docker-compose.yml index b971e83f..fc3e578c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,8 @@ services: wis2box-api: container_name: wis2box-api - image: ghcr.io/wmo-im/wis2box-api:latest + image: wis2box-api:issue-806 + #image: ghcr.io/wmo-im/wis2box-api:latest restart: always env_file: - wis2box.env @@ -40,6 +41,8 @@ services: test: ["CMD", "curl", "-f", "http://localhost/oapi/admin/resources"] interval: 5s retries: 100 + volumes: + - ${WIS2BOX_HOST_DATADIR}/mappings:/data/wis2box/mappings:ro minio: container_name: wis2box-minio @@ -100,6 +103,8 @@ services: context: ./wis2box-broker env_file: - wis2box.env + volumes: + - mosquitto-config:/mosquitto/config wis2box-management: container_name: wis2box-management @@ -150,3 +155,4 @@ volumes: minio-data: auth-data: htpasswd: + mosquitto-config: diff --git a/docs/source/reference/running/data-pipeline-plugins.rst b/docs/source/reference/running/data-pipeline-plugins.rst index 00f2e849..08d604ac 100644 --- a/docs/source/reference/running/data-pipeline-plugins.rst +++ b/docs/source/reference/running/data-pipeline-plugins.rst @@ -32,7 +32,21 @@ A typical csv2bufr plugin workflow definition would by defined as follows: csv: - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR - template: /data/wis2box/synop_bufr.json # locally created csv2bufr mapping (located in $WIS2BOX_HOST_DATADIR) + template: aws-template # using one of the built-in templates + notify: true # trigger GeoJSON publishing for API and UI + file-pattern: '^.*\.csv$' + +The default templates are defined by the `csv2bufr-templates`_ repository. + +In the case the user wants to use a custom template, the template should be located in the ``$WIS2BOX_HOST_DATADIR/mappings`` directory. + +The plugin configuration would then be defined as follows: + +.. code-block:: yaml + + csv: + - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR + template: /data/wis2box/mappings/my_own_template.json # locally created csv2bufr mapping (located in $WIS2BOX_HOST_DATADIR/mappings) notify: true # trigger GeoJSON publishing for API and UI file-pattern: '^.*\.csv$' @@ -146,5 +160,7 @@ For example, to publish GRIB2 data matching the file-pattern ``^.*_(\d{8})\d{2}. See :ref:`data-mappings` for a full example data mapping configuration. .. _`csv2bufr`: https://csv2bufr.readthedocs.io +.. _`csv2bufr-templates`: https://github.com/wmo-im/csv2bufr-templates .. _`bufr2geojson`: https://github.com/wmo-im/bufr2geojson .. _`synop2bufr`: https://synop2bufr.readthedocs.io + diff --git a/docs/source/user/data-ingest.rst b/docs/source/user/data-ingest.rst index 90705e8f..2a9d4062 100644 --- a/docs/source/user/data-ingest.rst +++ b/docs/source/user/data-ingest.rst @@ -40,7 +40,7 @@ The wis2box provides 3 types of built-in plugins to publish data in BUFR format: * `bufr2bufr` : the input is received in BUFR format and split by subset, where each subset is published as a separate bufr message * `synop2bufr` : the input is received in `FM-12 SYNOP format `_ and converted to BUFR format. The year and month are extracted from the file pattern -* `csv2bufr` : the input is received in csv format and converted to BUFR format +* `csv2bufr` : the input is received in csv format and converted to BUFR format, a mapping template is used to convert the csv columns to BUFR encoded values. Custom mapping templates need to be placed in the ``$WIS2BOX_HOST_DATADIR/mappings`` directory. See :ref:`csv2bufr-templates` for examples of mapping templates. To publish data for other data formats you can use the 'Universal' plugin, which will pass through the data without any conversion. Please note that you will need to ensure that the date timestamp can be extracted from the file pattern when using this plugin.