From bdb2b42020a70ce90f151854ee1a91aa3f8f36b4 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 28 Oct 2024 08:25:35 -0400 Subject: [PATCH] add documentation (#24) --- .github/workflows/docs.yml | 37 ++++++++++++++ .readthedocs.yaml | 16 +++++++ docs/Makefile | 20 ++++++++ docs/requirements.txt | 3 ++ docs/source/_static/favicon.ico | Bin 0 -> 1150 bytes docs/source/_static/wmo-logo.svg | 1 + docs/source/administration.rst | 48 +++++++++++++++++++ docs/source/architecture.rst | 55 +++++++++++++++++++++ docs/source/conf.py | 79 ++++++++++++++++++++++++++++++ docs/source/configuration.rst | 80 +++++++++++++++++++++++++++++++ docs/source/contributing.rst | 10 ++++ docs/source/index.rst | 35 ++++++++++++++ docs/source/installation.rst | 29 +++++++++++ docs/source/introduction.rst | 26 ++++++++++ docs/source/license.rst | 16 +++++++ 15 files changed, 455 insertions(+) create mode 100644 .github/workflows/docs.yml create mode 100644 .readthedocs.yaml create mode 100644 docs/Makefile create mode 100644 docs/requirements.txt create mode 100644 docs/source/_static/favicon.ico create mode 100644 docs/source/_static/wmo-logo.svg create mode 100644 docs/source/administration.rst create mode 100644 docs/source/architecture.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/configuration.rst create mode 100644 docs/source/contributing.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/installation.rst create mode 100644 docs/source/introduction.rst create mode 100644 docs/source/license.rst diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..c51a33a --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,37 @@ +name: Build documentation ⚙️ + +on: + push: + branches: + - master + paths: + - 'docs/**' + pull_request: + branches: + - master + paths: + - 'docs/**' + release: + types: + - released + +jobs: + main: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - python-version: '3.10' + steps: + - uses: actions/checkout@master + - uses: actions/setup-python@v5 + name: Setup Python ${{ matrix.python-version }} + with: + python-version: ${{ matrix.python-version }} + - name: Install requirements 📦 + run: | + #pip3 install -r requirements.txt + pip3 install -r docs/requirements.txt + - name: build docs 🏗️ + run: cd docs && make html + diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..187fcac --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +sphinx: + configuration: docs/source/conf.py + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..ad5c453 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +sphinx +sphinx_rtd_theme +readthedocs-sphinx-search diff --git a/docs/source/_static/favicon.ico b/docs/source/_static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..689bd6f93cac1c8044f2f256f1e28496a23becb2 GIT binary patch literal 1150 zcmb7EX-HI27=42DCkUyKeq>7FLQxo6NkxlRmgQblqLqqfrBjYu%b4jnxu=YdntPf{ z<`!v=OQX4DP2y7Mm{ykN5}I0>R3@i;uZ);K1mEMn``-QTcfRwTdw3kDLT)uRjy$zE zH&u>P=QvJ}46;O4X4lEYaU{0zay8(f9G=FjAUv&)YsWMp=9%H@QB4T#^bzK?69Q{( z1fJRgkKN1X_AB$~eCxy}ylZQ~=bm=R9>k*JUN{ncZILRlhb%7^b%l2^)Z2v?SrNPr zsQ-uIX}l6|n`_CaL7~_W*UYsbu+nB+{B3mMZ=(a>BWoeA$VEp}1)9n-;kI3sVO%g; zhK?7N_|n&d+J_P(_}CEM>lojVvpY}_FsL7YGe4*V|!b?H0 z^ALK@2wB0di1Rdu$lieP7USplaf}Z1B2(y$+h?{hUAkV$NUx2cdR6DdpiBM~52CJ+ z`U&_uJ&h-+B1rfa_@rpTcef+P-IVm9i(rCVnjFM9X9PMjJ>I`=BD`%O%?QQR z5NDrkcLbTQtqm%R>=CljZ7Y`y!be`3l z{_`6Y|Ht?U=vAH`!s2(_Oc2kvWM@@*KMHNn9Y(&5Z_f&NA6i2^ zrU4P(9{Dmk%HREhd}s!>{WB<1Od-8k3W2jJ^AXj-)lg+FUx_n+j*SdLWA`wUo)03} s``n^?=jSQkJ&e_nAqfHhq66IbtzL3I#Su7~fOlAH(cFdqgQb4|1F(v3W&i*H literal 0 HcmV?d00001 diff --git a/docs/source/_static/wmo-logo.svg b/docs/source/_static/wmo-logo.svg new file mode 100644 index 0000000..d0d543b --- /dev/null +++ b/docs/source/_static/wmo-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/source/administration.rst b/docs/source/administration.rst new file mode 100644 index 0000000..6fcbca9 --- /dev/null +++ b/docs/source/administration.rst @@ -0,0 +1,48 @@ +.. _administration: + +Administration +============== + +Docker Compose +-------------- + +As a Docker Compose based application, ``wis2-gdc`` uses the ``docker compose`` command to manage services. The Docker Compose setups are defined in the following configurations: + +- ``docker-compose.yml`` +- ``docker-compose.override.yml`` + +To adjust service ports, edit ``docker-compose.override.yml`` accordingly. + +As a convenience, the ``Makefile`` in the root directory provides shortcuts to manage the Docker Compose setup (encapsulating the various ``docker compose`` commands). + +.. code-block:: bash + + # build all images + make build + + # build all images (no cache) + make force-build + + # start all containers + make up + + # reinitialize backend + make reinit-backend + + # start all containers in dev mode + make dev + + # view all container logs in realtime + make logs + + # login to the wis2-gdc-management container + make login + + # restart all containers + make restart + + # shutdown all containers + make down + + # remove all volumes + make rm diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst new file mode 100644 index 0000000..22cee46 --- /dev/null +++ b/docs/source/architecture.rst @@ -0,0 +1,55 @@ +.. _architecture: + +Architecture +============ + +``wis2-gdc`` is implemented in the spirit of the `Twelve-Factor App methodology`_. + +``wis2-gdc`` is a `Docker`_ and `Python`_-based platform in support of WIS2 metadata +management, publication, and discovery. + +High level system context +------------------------- + +The following diagram provides a high level overview of the main functions +of ``wis2-gdc``: + +.. figure:: ../architecture/c4.container.png + :scale: 70% + :alt: wis2-gdc architecture + :align: center + +How ``wis2-gdc`` works +---------------------- + +As a containerized solution, ``wis2-gdc`` provides functionality via the following services: + +* **wis2-gdc-management**: ingests, validates and publishes discovery metadata published from a WIS2 Global Broker instance +* **wis2-gdc-api**: API powered by `pygeoapi`_ +* **wis2-gdc-monitoring**: monitoring + + * **metrics-collector**: metrics collector + * **prometheus**: metrics scraper + * **grafana**: analytics and visualization +* **wis2-gdc-broker**: MQTT broker +* **wis2-gdc-backend**: API search engine backend (default Elasticsearch) + +``wis2-gdc`` is primarily an event-driven system, also providing interactive functionality for managing WIS2 metadata if required. + +Workflows +^^^^^^^^^ + +``wis2-gdc`` starts up by connecting to a WIS2 Global Broker (GB), subscribing to notifications for WIS2 metadata (WMO Core Metadata Profile [WCMP2]). On receipt of WIS2 Notification Messages (WNM) for metadata, ``wis2-gdc`` will ingest, validate and publish a WCMP2 record to its catalogue API. In addition, WCMP2 update and deletion is supported with the appropriate WNM. + +The ``wis2-gdc`` monitoring capability collects and provides metrics on WCMP2 that is scraped by the WIS2 Global Monitor (GM). + +``wis2-gdc`` also provides its own MQTT broker that provides WCMP2 compliance and key performance indicator (KPI) reports. The WIS2 GB subscribes to the ``wis2-gdc`` broker in order to publish these reports for data providers in support of quality assessment, scoring and corrective action. The ``wis2-gdc`` broker also acts as an internal message bus for internal event handling. + +The ``wis2-gdc`` API provides an OGC API - Records endpoint that is OGC compliant. The API provides search engine capability for WIS2 metadata, also providing OGC API - Processes functionality for WCMP2 validation and KPI quality assessment. + + + +.. _`Twelve-Factor App methodology`: https://12factor.net +.. _`Docker`: https://www.docker.com +.. _`Python`: https://www.python.org +.. _`pygeoapi`: https://pygeoapi.io diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..c8226c5 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,79 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2024 Angelos Tzotsos +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'wis2-gdc' +author = 'Tom Kralidis' +license = 'This work is licensed under a Creative Commons Attribution 4.0 International License' # noqa +release = '0.1.0' +copyright = '2023-present, ' + author + ' ' + license + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.doctest', + 'sphinx.ext.githubpages', + 'sphinx.ext.intersphinx'] + +today_fmt = '%Y-%m-%d' + +templates_path = ['_templates'] +exclude_patterns = [] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] + +html_favicon = '_static/favicon.ico' + +#html_sidebars = { +# '**': [ +# 'relations.html', # needs 'show_related': True theme option to display +# 'searchbox.html', +# 'indexsidebar.html'] +#} + +pygments_style = 'sphinx' + + diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst new file mode 100644 index 0000000..85d5f1f --- /dev/null +++ b/docs/source/configuration.rst @@ -0,0 +1,80 @@ +.. _configuration: + +Configuration +============= + +``wis2-gdc`` configuration is driven by the following environment variables, which are managed in ``wis2-gdc.env``: + +.. csv-table:: Environment variables + :widths: 30 30 30 + :header: Name,Description,Default + + ``WIS2_GDC_LOGGING_LEVEL``,logging level as per the standard `Python logging levels`_,``ERROR`` + ``WIS2_GDC_API_URL``,public URL of the GDC API,``http://localhost`` + ``WIS2_GDC_API_URL_DOCKER``,internal Docker URL of the API,``http://wis2-gdc-api`` + ``WIS2_GDC_BACKEND_TYPE``,API backend type,``Elasticsearch`` + ``WIS2_GDC_BACKEND_CONNECTION``,API backend connection,``http://wis2-gdc-backend:9200/wis2-discovery-metadata`` + ``WIS2_GDC_BROKER_URL``,URL of the GDC broker,``mqtt://wis2-gdc:wis2-gdc@wis2-gdc-broker:1883`` + ``WIS2_GDC_CENTRE_ID``,centre identifier of the GDC,``ca-eccc-msc-global-discovery-catalogue`` + ``WIS2_GDC_COLLECTOR_URL``,URL of metrics collector,``http://wis2-gdc-metrics-collector:8006`` + ``WIS2_GDC_GB``,WIS2 Global Broker that the GDC connects to,``mqtts://everyone:everyone@globalbroker.meteo.fr:8883`` + ``WIS2_GDC_GB_TOPIC``,WIS2 topic that the GDC subscribes to,``cache/a/wis2/+/metadata/#`` + ``WIS2_GDC_METADATA_ARCHIVE_ZIPFILE``,filepath to the metadata archive zipfile,``/data/wis2-gdc-archive.zip`` + ``WIS2_GDC_PUBLISH_REPORTS``,whether the GDC should publish ETS and KPI reports,``true`` + ``WIS2_GDC_REJECT_ON_FAILING_ETS``,whether the GDC should stop ingest based on on failing record,``true`` + ``WIS2_GDC_RUN_KPI``,whether the GDC should run KPI as part of ingest,``false`` + +Global Broker environment variables +----------------------------------- + +WIS2 Global Broker environment variables are defined as comma-separated values (centre=id,url,centre-name). ``wis2-gdc`` allows for 1..n Global Broker environment variables as required. + +.. note:: + + - the naming convention is ``WIS_GDC_GB_LINK_