diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml new file mode 100644 index 0000000..dba15fe --- /dev/null +++ b/.github/workflows/flake8.yml @@ -0,0 +1,24 @@ +name: flake8 + +on: + [ push, pull_request ] + +jobs: + flake8_py3: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + name: setup Python + with: + python-version: 3.8 + - name: Checkout pywis-pubsub + uses: actions/checkout@master + - name: Install flake8 + run: pip install flake8 + - name: Run flake8 + uses: suo/flake8-github-action@releases/v1 + with: + checkName: 'flake8_py3' # NOTE: this needs to be the same as the job name + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1080ea9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,136 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# wis2-grep +local.env +local.yml + +# architecture diagram backups +*.bkp diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ea77198 --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +DOCKER_COMPOSE_ARGS=--project-name wis2-grep --file docker-compose.yml --file docker-compose.override.yml + +build: + docker compose $(DOCKER_COMPOSE_ARGS) build + +up: + docker compose $(DOCKER_COMPOSE_ARGS) up --detach + +dev: + docker compose $(DOCKER_COMPOSE_ARGS) --file docker-compose.dev.yml up + +login: + docker exec -it wis2-grep-management /bin/bash + +down: + docker compose $(DOCKER_COMPOSE_ARGS) down + +restart: down up + +force-build: + docker compose $(DOCKER_COMPOSE_ARGS) build --no-cache + +logs: + docker compose $(DOCKER_COMPOSE_ARGS) logs --follow + +clean: + docker system prune -f + docker volume prune -f + +rm: + docker volume rm $(shell docker volume ls --filter name=wis2-grep -q) + +.PHONY: build up dev login down restart force-build logs rm clean diff --git a/README.md b/README.md index 158bfd2..45c68fd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,141 @@ +[![flake8](https://github.com/wmo-im/wis2-grep/workflows/flake8/badge.svg)](https://github.com/wmo-im/wis2-grep/actions) + # wis2-grep -WIS2 Global Replay Service Reference Implementation + +wis2-grep is a Reference Implementation of a WIS2 Global Replay Service. + +WIS2 Global Replay Service C4 component diagram + +## Workflow + +- connects to a WIS2 Global Broker, subscribed to the following topic: + - `origin/a/wis2/#` +- on notification messages + - publish to a WIS2 Global Replay Service (OGC API - Features) using one of the supported transaction backends: + - [OGC API - Features - Part 4: Create, Replace, Update and Delete](https://docs.ogc.org/DRAFTS/20-002.html) + - Elasticsearch direct (default) + +## Installation + +### Requirements +- Python 3 +- [virtualenv](https://virtualenv.pypa.io) + +### Dependencies +Dependencies are listed in [requirements.txt](requirements.txt). Dependencies +are automatically installed during pywis-pubsub installation. + +### Installing wis2-grep + +```bash +# setup virtualenv +python3 -m venv --system-site-packages wis2-grep +cd wis2-grep +source bin/activate + +# clone codebase and install +git clone https://github.com/wmo-im/wis2-grep.git +cd wis2-grep-management +python3 setup.py install +``` + +## Running + +```bash +# setup environment and configuration +cp wis2-grep.env local.env +vim local.env # update accordingly + +source local.env + +# setup pywis-pubsub - sync WIS2 notification schema +pywis-pubsub schema sync + +# setup backend +wis2-grep setup + +# teardown backend +wis2-grep teardown + +# connect to Global Broker +# notifications will automatically trigger wis2-grep to publish +# WNM to the API identified in wis2-grep.env (WIS2_GREP_GB) +pywis-pubsub subscribe --config pywis-pubsub.yml + +# loading notification messsage manually (single file) +wis2-grep register /path/to/wnm-file.json + +# loading notification messages manually (directory of .json files) +wis2-grep load /path/to/dir/of/wnm-files +``` + +### Docker + +The Docker setup uses Docker and Docker Compose to manage the following services: + +- **wis2-grep-api**: API powered by [pygeoapi](https://pygeoapi.io) +- **wis2-grep-management**: management service to publish notification messages published from a WIS2 Global Broker instance + - the default Global Broker connection is to NOAA. This can be modified in `wis2-grep.env` to point to a different Global Broker +- ** wis2-grep-backend**: API search engine backend (default Elasticsearch) + +See [`wis2-grep.env`](wis2-grep.env) for default environment variable settings. + +To adjust service ports, edit [`docker-compose.override.yml`](docker-compose.override.yml) accordingly. + +The [`Makefile`](Makefile) in the root directory provides options to manage the Docker Compose setup. + +```bash +# build all images +make build + +# build all images (no cache) +make force-build + +# start all containers +make up + +# start all containers in dev mode +make dev + +# view all container logs in realtime +make logs + +# login to the wis2-grep-management container +make login + +# restart all containers +make restart + +# shutdown all containers +make down + +# remove all volumes +make rm +``` + +## Development + +### Running Tests + +```bash +# install dev requirements +pip3 install -r requirements-dev.txt + +# run tests like this: +python3 tests/run_tests.py + +# or this: +python3 setup.py test +``` + +### Code Conventions + +* [PEP8](https://www.python.org/dev/peps/pep-0008) + +### Bugs and Issues + +All bugs, enhancements and issues are managed on [GitHub](https://github.com/wmo-im/wis2-grep/issues). + +## Contact + +* [Tom Kralidis](https://github.com/tomkralidis) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..9756662 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,29 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +services: + wis2-grep-backend: + ports: + - 9200:9200 + wis2-grep-management: + volumes: + - wis2-grep-management-data:/data + - ./wis2-grep-management/wis2_grep:/app/wis2_grep diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..95a4594 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,25 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +services: + wis2-grep-api: + ports: + - 80:80 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..34e186f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,100 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +services: + wis2-grep-backend: + container_name: wis2-grep-backend + image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2 + restart: always + environment: + - discovery.type=single-node + - discovery.seed_hosts=[] + - node.name=elasticsearch-01 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - cluster.name=es-wis2-grep + - xpack.security.enabled=false + mem_limit: 1.5g + memswap_limit: 1.5g + volumes: + - wis2-grep-backend-data:/usr/share/elasticsearch/data:rw + # ulimits: + # nofile: + # soft: 524288 + # hard: 524288 + # memlock: + # soft: -1 + # hard: -1 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200"] + interval: 5s + retries: 100 + networks: + - wis2-grep-net + + wis2-grep-management: + container_name: wis2-grep-management + build: + context: ./wis2-grep-management/ + env_file: + - wis2-grep.env + environment: + - WIS2_GREP_API_URL_DOCKER=http://wis2-grep-api + depends_on: + wis2-grep-backend: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://wis2-grep-backend:9200/wis2-notification-messages"] + interval: 5s + retries: 3 + volumes: + - wis2-grep-management-data:/data + restart: always + command: ["pywis-pubsub", "subscribe", "--config", "/app/docker/pywis-pubsub.yml", "--verbosity", "DEBUG"] + networks: + - wis2-grep-net + + wis2-grep-api: + container_name: wis2-grep-api + build: + context: ./wis2-grep-api/ + image: geopython/pygeoapi:latest + depends_on: + wis2-grep-management: + condition: service_healthy + volumes: + - wis2-grep-management-data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost/conformance"] + interval: 5s + retries: 3 + env_file: + - wis2-grep.env + restart: unless-stopped + networks: + - wis2-grep-net + +volumes: + wis2-grep-backend-data: + wis2-grep-management-data: + +networks: + wis2-grep-net: diff --git a/docs/architecture/README.md b/docs/architecture/README.md new file mode 100644 index 0000000..81325e7 --- /dev/null +++ b/docs/architecture/README.md @@ -0,0 +1,11 @@ +# Architecture diagrams + +The architecture diagrams in this directory are designed using the [C4 model](https://c4model.com) for visualising software architecture. + +The diagrams are saved as an editable `.png` file, for easy default viewing on GitHub, showing the last active diagram when saved. Any PNG +viewer will also render the diagram in the same way. + +The diagrams can be updated using [diagrams.net](https://diagrams.net) in the following manner: + +- open files on this GitHub repository directly in diagrams.net +- clone this repository, edit/save using the diagrams.net desktop application ([download](https://github.com/jgraph/drawio-desktop/releases)), and commit/push files to GitHub diff --git a/docs/architecture/c4.container.png b/docs/architecture/c4.container.png new file mode 100644 index 0000000..2713a31 Binary files /dev/null and b/docs/architecture/c4.container.png differ diff --git a/wis2-grep-api/Dockerfile b/wis2-grep-api/Dockerfile new file mode 100644 index 0000000..0ec43e0 --- /dev/null +++ b/wis2-grep-api/Dockerfile @@ -0,0 +1,38 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +FROM geopython/pygeoapi:latest + +LABEL maintainer="Tom Kralidis " + +ENV PYGEOAPI_CONFIG=/pygeoapi/local.config.yml +ENV PYGEOAPI_OPENAPI=/pygeoapi/local.openapi.yml + +RUN apt-get update && \ + apt-get install -y curl + +COPY ./app.py /pygeoapi/pygeoapi/app.py +COPY ./docker/wis2-grep-api.yml /pygeoapi/local.config.yml +COPY ./docker/entrypoint.sh /app/docker/wis2-grep-api/entrypoint.sh + +RUN chmod +x /app/docker/wis2-grep-api/entrypoint.sh + +ENTRYPOINT [ "/app/docker/wis2-grep-api/entrypoint.sh" ] diff --git a/wis2-grep-api/app.py b/wis2-grep-api/app.py new file mode 100644 index 0000000..3905bc0 --- /dev/null +++ b/wis2-grep-api/app.py @@ -0,0 +1,35 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +from flask import Flask +from pygeoapi.flask_app import BLUEPRINT as pygeoapi_blueprint + + +app = Flask(__name__, static_url_path='/static') +app.url_map.strict_slashes = False + +app.register_blueprint(pygeoapi_blueprint, url_prefix='/') + +try: + from flask_cors import CORS + CORS(app) +except ImportError: # CORS needs to be handled by upstream server + pass diff --git a/wis2-grep-api/docker/entrypoint.sh b/wis2-grep-api/docker/entrypoint.sh new file mode 100644 index 0000000..32855e0 --- /dev/null +++ b/wis2-grep-api/docker/entrypoint.sh @@ -0,0 +1,83 @@ +#!/bin/bash +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +# pygeoapi entry script + +echo "START /entrypoint.sh" + +set +e + +# gunicorn env settings with defaults +SCRIPT_NAME="/" +CONTAINER_NAME="wis2-grep-api" +CONTAINER_HOST=${CONTAINER_HOST:=0.0.0.0} +CONTAINER_PORT=${CONTAINER_PORT:=80} +WSGI_WORKERS=${WSGI_WORKERS:=4} +WSGI_WORKER_TIMEOUT=${WSGI_WORKER_TIMEOUT:=6000} +WSGI_WORKER_CLASS=${WSGI_WORKER_CLASS:=gevent} + +# What to invoke: default is to run gunicorn server +entry_cmd=${1:-run} + +# Shorthand +function error() { + echo "ERROR: $@" + exit -1 +} + +# Workdir +cd /pygeoapi + +# Lock all Python files (for gunicorn hot reload) +find . -type f -name "*.py" | xargs chmod -R 0444 + +echo "Trying to generate OpenAPI document" +pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI} + +[[ $? -ne 0 ]] && error "ERROR: OpenAPI document could not be generated" + +echo "openapi.yml generated continue to pygeoapi" + +case ${entry_cmd} in + # Run pygeoapi server + run) + # SCRIPT_NAME should not have value '/' + [[ "${SCRIPT_NAME}" = '/' ]] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" + + echo "Start gunicorn name=${CONTAINER_NAME} on ${CONTAINER_HOST}:${CONTAINER_PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" + exec gunicorn --workers ${WSGI_WORKERS} \ + --worker-class=${WSGI_WORKER_CLASS} \ + --timeout ${WSGI_WORKER_TIMEOUT} \ + --name=${CONTAINER_NAME} \ + --bind ${CONTAINER_HOST}:${CONTAINER_PORT} \ + --reload \ + --reload-extra-file ${PYGEOAPI_CONFIG} \ + pygeoapi.app:app \ + --access-logfile '-' + ;; + *) + error "unknown command arg: must be run (default)" + ;; +esac + +echo "END /entrypoint.sh" + diff --git a/wis2-grep-api/docker/wis2-grep-api.yml b/wis2-grep-api/docker/wis2-grep-api.yml new file mode 100644 index 0000000..33d00a0 --- /dev/null +++ b/wis2-grep-api/docker/wis2-grep-api.yml @@ -0,0 +1,101 @@ +server: + bind: + host: 0.0.0.0 + port: 5000 + url: ${WIS2_GREP_API_URL} + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + cors: true + pretty_print: true + limit: 10 + # templates: /path/to/templates + map: + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: 'Map data © OpenStreetMap contributors' + +logging: + level: ${WIS2_GREP_LOGGING_LEVEL} + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: WMO WIS2 Global Replay instance + description: WMO WIS2 Global Replay instance + keywords: + - wmo + - wis2 + - replay + - notifications + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0 + url: https://github.com/wmo-im/wis2-grep + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0 + provider: + name: WIS2 Global Replay Service API provider name + url: https://github.com/wmo-im/wis2-grep + contact: + name: Firstname Lastname + position: Position name + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: pygeoapi@lists.osgeo.org + url: https://github.com/wmo-im/wis2-grep + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + wis2-notification-messages: + type: collection + title: WIS2 notification messages + description: WIS2 notification messages + keywords: [wmo, wis2, notifications] + crs: + - CRS84 + links: + - type: application/geo+json + rel: items + title: Notifications from Météo-France, Global Broker Service + href: mqtts://everyone:everyone@globalbroker.meteo.fr:8883 + channel: 'origin/a/wis2/#' + length: -1 + - type: application/geo+json + rel: items + title: Notifications from China Meteorological Agency, Global Broker Service + href: mqtts://everyone:everyone@gb.wis.cma.cn:8883 + channel: 'origin/a/wis2/#' + length: -1 + - type: application/geo+json + rel: items + title: Notifications from National Oceanic and Atmospheric Administration, National Weather Service, Global Broker Service + href: mqtts://everyone:everyone@wis2globalbroker.nws.noaa.gov:8883 + channel: 'origin/a/wis2/#' + length: -1 + - type: application/geo+json + rel: items + title: Notifications from Instituto Nacional de Meteorologia (Brazil), Global Broker Service + href: mqtts://everyone:everyone@globalbroker.inmet.gov.br:8883 + channel: 'origin/a/wis2/#' + length: -1 + - type: text/html + rel: canonical + title: WMO Information System (WIS) | World Meteorological Organization + href: https://community.wmo.int/en/activity-areas/wis + extents: + spatial: + bbox: [-180, -90, 180, 90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: ${WIS2_GREP_BACKEND_TYPE} + data: ${WIS2_GREP_BACKEND_CONNECTION} + id_field: id + time_field: pubtime diff --git a/wis2-grep-management/Dockerfile b/wis2-grep-management/Dockerfile new file mode 100644 index 0000000..4d83ba7 --- /dev/null +++ b/wis2-grep-management/Dockerfile @@ -0,0 +1,51 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +FROM ubuntu:focal + +LABEL maintainer="tomkralidis@gmail.com" + +ENV TZ="Etc/UTC" \ + DEBIAN_FRONTEND="noninteractive" \ + DEBIAN_PACKAGES="bash cron curl git python3-pip python3-setuptools vim" + +# copy the app +COPY ./ /app + +# add to crontab +COPY ./docker/wis2-grep-management.cron /etc/cron.d/wis2-grep-management.cron + +RUN apt-get update -y && \ + # install dependencies + apt-get install -y ${DEBIAN_PACKAGES} && \ + pip3 install --no-cache-dir -r /app/requirements-backend.txt && \ + # install wis2-grep + cd /app && \ + pip3 install -e . && \ + chmod +x /app/docker/entrypoint.sh && \ + # cleanup + apt autoremove -y && \ + apt-get -q clean && \ + rm -rf /var/lib/apt/lists/* && \ + chmod 0644 /etc/cron.d/wis2-grep-management.cron && \ + crontab /etc/cron.d/wis2-grep-management.cron + +ENTRYPOINT [ "/app/docker/entrypoint.sh" ] diff --git a/wis2-grep-management/MANIFEST.in b/wis2-grep-management/MANIFEST.in new file mode 100644 index 0000000..b2cc4f5 --- /dev/null +++ b/wis2-grep-management/MANIFEST.in @@ -0,0 +1 @@ +include README.md LICENSE requirements.txt diff --git a/wis2-grep-management/README.md b/wis2-grep-management/README.md new file mode 100644 index 0000000..4fe4fbc --- /dev/null +++ b/wis2-grep-management/README.md @@ -0,0 +1,3 @@ +# wis2-grep -management + +Python package to perform WIS2 Global Replay Service management functions. diff --git a/wis2-grep-management/docker/entrypoint.sh b/wis2-grep-management/docker/entrypoint.sh new file mode 100755 index 0000000..89ce1cb --- /dev/null +++ b/wis2-grep-management/docker/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/bash +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +# wis2-grep entry script + +echo "START /entrypoint.sh" + +echo "Caching WNM schema" +pywis-pubsub schema sync + +echo "Setting up notification message backend" +wis2-grep setup --yes + +echo "END /entrypoint.sh" +exec "$@" diff --git a/wis2-grep-management/docker/pywis-pubsub.yml b/wis2-grep-management/docker/pywis-pubsub.yml new file mode 100644 index 0000000..31b819c --- /dev/null +++ b/wis2-grep-management/docker/pywis-pubsub.yml @@ -0,0 +1,12 @@ +broker: ${WIS2_GREP_GB} + +subscribe_topics: + - ${WIS2_GREP_GB_TOPIC} + +qos: 1 + +verify_data: false + +validate_message: false + +hook: wis2_grep.hook.NotificationMessageHook diff --git a/wis2-grep-management/docker/wis2-grep-management.cron b/wis2-grep-management/docker/wis2-grep-management.cron new file mode 100644 index 0000000..2878605 --- /dev/null +++ b/wis2-grep-management/docker/wis2-grep-management.cron @@ -0,0 +1 @@ +19 * * * * su -c "wis2-grep clean" > /proc/1/fd/1 2>/proc/1/fd/2 diff --git a/wis2-grep-management/requirements-backend.txt b/wis2-grep-management/requirements-backend.txt new file mode 100644 index 0000000..c1cfe6b --- /dev/null +++ b/wis2-grep-management/requirements-backend.txt @@ -0,0 +1,2 @@ +elasticsearch +OWSLib diff --git a/wis2-grep-management/requirements-dev.txt b/wis2-grep-management/requirements-dev.txt new file mode 100644 index 0000000..ff3303f --- /dev/null +++ b/wis2-grep-management/requirements-dev.txt @@ -0,0 +1,3 @@ +flake8 +twine +wheel diff --git a/wis2-grep-management/requirements.txt b/wis2-grep-management/requirements.txt new file mode 100644 index 0000000..6963e06 --- /dev/null +++ b/wis2-grep-management/requirements.txt @@ -0,0 +1,2 @@ +click +pywis-pubsub diff --git a/wis2-grep-management/setup.py b/wis2-grep-management/setup.py new file mode 100644 index 0000000..c1f161d --- /dev/null +++ b/wis2-grep-management/setup.py @@ -0,0 +1,109 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +from pathlib import Path +import re +from setuptools import Command, find_packages, setup +import sys + + +class PyTest(Command): + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + import subprocess + errno = subprocess.call([sys.executable, 'tests/run_tests.py']) + raise SystemExit(errno) + + +def read(filename, encoding='utf-8'): + """read file contents""" + + fullpath = Path(__file__).resolve().parent / filename + + with fullpath.open() as fh: + contents = fh.read().strip() + return contents + + +def get_package_version(): + """get version from top-level package init""" + version_file = read('wis2_grep/__init__.py') + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError('Unable to find version string.') + + +LONG_DESCRIPTION = read('README.md') + +DESCRIPTION = 'wis2-grep is a Reference Implementation of a WIS2 Global Replay Service' # noqa + +MANIFEST = Path('MANIFEST') + +if MANIFEST.exists(): + MANIFEST.unlink() + + +setup( + name='wis2-grep', + version=get_package_version(), + description=DESCRIPTION.strip(), + long_description=LONG_DESCRIPTION, + long_description_content_type='text/markdown', + license='Apache Software License', + platforms='all', + keywords=' '.join([ + 'WIS2', + 'Global Replace Service', + 'notifications' + ]), + author='Tom Kralidis', + author_email='tomkralidis@gmail.com', + maintainer='Tom Kralidis', + maintainer_email='tomkraldis@gmail.com', + url='https://github.com/wmo-im/wis2-grep', + install_requires=read('requirements.txt').splitlines(), + packages=find_packages(), + include_package_data=True, + entry_points={ + 'console_scripts': [ + 'wis2-grep=wis2_grep:cli' + ] + }, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python' + ], + cmdclass={'test': PyTest} +) diff --git a/wis2-grep-management/wis2_grep/__init__.py b/wis2-grep-management/wis2_grep/__init__.py new file mode 100644 index 0000000..72f2efd --- /dev/null +++ b/wis2-grep-management/wis2_grep/__init__.py @@ -0,0 +1,39 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import click + +from wis2_grep.loader import load, setup, teardown + +__version__ = '0.1.dev0' + + +@click.group() +@click.version_option(version=__version__) +def cli(): + """WIS2 Replace Service management utilities""" + + pass + + +cli.add_command(setup) +cli.add_command(teardown) +cli.add_command(load) diff --git a/wis2-grep-management/wis2_grep/backend/__init__.py b/wis2-grep-management/wis2_grep/backend/__init__.py new file mode 100644 index 0000000..763e217 --- /dev/null +++ b/wis2-grep-management/wis2_grep/backend/__init__.py @@ -0,0 +1,29 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +from wis2_grep.backend.elastic import ElasticsearchBackend +from wis2_grep.backend.ogcapi_features import OGCAPIFeaturesBackend + + +BACKENDS = { + 'Elasticsearch': ElasticsearchBackend, + 'OGCAPIFeatures': OGCAPIFeaturesBackend +} diff --git a/wis2-grep-management/wis2_grep/backend/base.py b/wis2-grep-management/wis2_grep/backend/base.py new file mode 100644 index 0000000..26631de --- /dev/null +++ b/wis2-grep-management/wis2_grep/backend/base.py @@ -0,0 +1,77 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +from abc import ABC, abstractmethod +import logging + +LOGGER = logging.getLogger(__name__) + + +class BaseBackend(ABC): + def __init__(self, defs): + self.defs = defs + + @abstractmethod + def setup(self) -> None: + """ + Setup a backend + + :returns: `None` + """ + + raise NotImplementedError() + + @abstractmethod + def teardown(self) -> None: + """ + Tear down a backend + + :returns: `None` + """ + + raise NotImplementedError() + + @abstractmethod + def save(self, record: dict) -> None: + """ + Upsert a resource to a backend + + :param payload: `dict` of resource + + :returns: `None` + """ + + raise NotImplementedError() + + @abstractmethod + def exists(self, identifier: str) -> bool: + """ + Querying whether a record exists in a backend + + :param identifier: `str` of record identifier + + :returns: `bool` of whether record exists in backend + """ + + raise NotImplementedError() + + def __repr__(self): + return '' diff --git a/wis2-grep-management/wis2_grep/backend/elastic.py b/wis2-grep-management/wis2_grep/backend/elastic.py new file mode 100644 index 0000000..e7d0f0f --- /dev/null +++ b/wis2-grep-management/wis2_grep/backend/elastic.py @@ -0,0 +1,170 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import logging +from urllib.parse import urlparse + +from elasticsearch import Elasticsearch, NotFoundError + +from wis2_grep.backend.base import BaseBackend + +LOGGER = logging.getLogger(__name__) + + +class ElasticsearchBackend(BaseBackend): + + from urllib.parse import urlparse + from elasticsearch import Elasticsearch + + def __init__(self, defs): + super().__init__(defs) + + # default index settings + self.ES_SETTINGS = { + 'settings': { + 'number_of_shards': 1, + 'number_of_replicas': 0 + }, + 'mappings': { + 'properties': { + 'id': { + 'type': 'text', + 'fields': { + 'raw': { + 'type': 'keyword' + } + } + }, + 'geometry': { + 'type': 'geo_shape' + }, + 'time': { + 'properties': { + 'interval': { + 'type': 'date', + 'null_value': '1850', + 'format': 'year||year_month||year_month_day||date_time||t_time||t_time_no_millis', # noqa + 'ignore_malformed': True + } + } + }, + 'properties': { + 'properties': { + 'type': { + 'type': 'text', + 'fields': { + 'raw': { + 'type': 'keyword' + } + } + }, + 'title': { + 'type': 'text', + 'fields': { + 'raw': { + 'type': 'keyword' + } + } + }, + 'description': { + 'type': 'text', + 'fields': { + 'raw': { + 'type': 'keyword' + } + } + }, + 'wmo:dataPolicy': { + 'type': 'text', + 'fields': { + 'raw': { + 'type': 'keyword' + } + } + } + } + } + } + } + } + + self.url_parsed = urlparse(self.defs.get('connection')) + self.index_name = self.url_parsed.path.lstrip('/') + + url2 = f'{self.url_parsed.scheme}://{self.url_parsed.netloc}' + + if self.url_parsed.port is None: + LOGGER.debug('No port found; trying autodetect') + port = None + if self.url_parsed.scheme == 'http': + port = 80 + elif self.url_parsed.scheme == 'https': + port = 443 + if port is not None: + url2 = f'{self.url_parsed.scheme}://{self.url_parsed.netloc}:{port}' # noqa + + if self.url_parsed.path.count('/') > 1: + LOGGER.debug('ES URL has a basepath') + basepath = self.url_parsed.path.split('/')[1] + self.index_name = self.url_parsed.path.split('/')[-1] + url2 = f'{url2}/{basepath}/' + + LOGGER.debug(f'ES URL: {url2}') + LOGGER.debug(f'ES index: {self.index_name}') + + settings = { + 'hosts': [url2], + 'retry_on_timeout': True, + 'max_retries': 10, + 'timeout': 30 + } + + if self.url_parsed.username and self.url_parsed.password: + settings['http_auth'] = ( + self.url_parsed.username, self.url_parsed.password) + + LOGGER.debug(f'Settings: {settings}') + self.es = Elasticsearch(**settings) + + def setup(self) -> None: + self.teardown() + LOGGER.debug(f'Creating index {self.index_name}') + self.es.indices.create(index=self.index_name, body=self.ES_SETTINGS) + + def teardown(self) -> None: + if self.es.indices.exists(index=self.index_name): + LOGGER.debug(f'Deleting index {self.index_name}') + self.es.indices.delete(index=self.index_name) + + def save(self, record: dict) -> None: + LOGGER.debug(f"Indexing record {record['id']}") + self.es.index(index=self.index_name, id=record['id'], body=record) + + def exists(self, identifier: str) -> bool: + LOGGER.debug(f'Querying Replay API for id {identifier}') + try: + _ = self.es.get(index=self.index_name, id=identifier) + return True + except NotFoundError: + return False + + def __repr__(self): + return '' diff --git a/wis2-grep-management/wis2_grep/backend/ogcapi_features.py b/wis2-grep-management/wis2_grep/backend/ogcapi_features.py new file mode 100644 index 0000000..8d05c38 --- /dev/null +++ b/wis2-grep-management/wis2_grep/backend/ogcapi_features.py @@ -0,0 +1,69 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import logging +import json + +from owslib.ogcapi.features import Features + +from wis2_grep import env +from wis2_grep.backend.base import BaseBackend + +LOGGER = logging.getLogger(__name__) + + +class OGCAPIFeaturesBackend(BaseBackend): + + def __init__(self, defs): + super().__init__(defs) + + self.conn = Features(env.API_URL) + self.collection = 'notification-messsages' + + def save(self): + + ttype = 'create' + + try: + _ = self.conn.get_collection_item(self.metadata['id']) + ttype = 'update' + except Exception: + pass + + payload = json.dumps(self.metadata) + + if ttype == 'create': + LOGGER.debug('Adding new notification to collection') + _ = self.conn.get_collection_create(self.collection, payload) + elif ttype == 'update': + LOGGER.debug('Updating existing notification in collection') + _ = self.conn.get_collection_update(self.collection, payload) + + def exists(self, identifier: str) -> bool: + LOGGER.debug(f'Querying Replay API for id {identifier}') + try: + _ = self.conn.collection_item(self.collection, identifier) + return True + except RuntimeError: + return False + + def __repr__(self): + return '' diff --git a/wis2-grep-management/wis2_grep/env.py b/wis2-grep-management/wis2_grep/env.py new file mode 100644 index 0000000..46a160c --- /dev/null +++ b/wis2-grep-management/wis2_grep/env.py @@ -0,0 +1,58 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import os +from typing import Any + + +def str2bool(value: Any) -> bool: + """ + helper function to return Python boolean + type (source: https://stackoverflow.com/a/715468) + + :param value: value to be evaluated + + :returns: `bool` of whether the value is boolean-ish + """ + + value2 = False + + if isinstance(value, bool): + value2 = value + else: + value2 = value.lower() in ('yes', 'true', 't', '1', 'on') + + return value2 + + +API_URL = os.environ.get('WIS2_GREP_API_URL') +API_URL_DOCKER = os.environ.get('WIS2_GREP_API_URL_DOCKER') +BACKEND_TYPE = os.environ.get('WIS2_GREP_BACKEND_TYPE') +BACKEND_CONNECTION = os.environ.get('WIS2_GREP_BACKEND_CONNECTION') +CENTRE_ID = os.environ.get('WIS2_GREP_CENTRE_ID') +GB = os.environ.get('WIS2_GREP_GB') +GB_TOPIC = os.environ.get('WIS2_GREP_GB_TOPIC') + +print(API_URL, API_URL_DOCKER, BACKEND_TYPE, BACKEND_CONNECTION, CENTRE_ID, GB, GB_TOPIC) + +if None in [API_URL, API_URL_DOCKER, BACKEND_TYPE, BACKEND_CONNECTION, + CENTRE_ID, GB, GB_TOPIC]: + raise EnvironmentError('Environment variables not set!') diff --git a/wis2-grep-management/wis2_grep/hook.py b/wis2-grep-management/wis2_grep/hook.py new file mode 100644 index 0000000..0553512 --- /dev/null +++ b/wis2-grep-management/wis2_grep/hook.py @@ -0,0 +1,39 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import logging + +from pywis_pubsub.hook import Hook + +from wis2_grep.loader import Loader + +LOGGER = logging.getLogger(__name__) + + +class NotificationMessageHook(Hook): + def execute(self, topic: str, msg_dict: dict) -> None: + LOGGER.debug('Notification message hook execution begin') + loader = Loader() + loader.load(msg_dict, topic) + LOGGER.debug('Notification message hook execution end') + + def __repr__(self): + return '' diff --git a/wis2-grep-management/wis2_grep/loader.py b/wis2-grep-management/wis2_grep/loader.py new file mode 100644 index 0000000..1032bcd --- /dev/null +++ b/wis2-grep-management/wis2_grep/loader.py @@ -0,0 +1,145 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import json +import logging +from pathlib import Path +from typing import Union + +import click + +from pywis_pubsub import cli_options + +from wis2_grep.backend import BACKENDS +from wis2_grep.env import BACKEND_TYPE, BACKEND_CONNECTION + +LOGGER = logging.getLogger(__name__) + + +class Loader: + def __init__(self): + """ + Initializer + + :returns: `wis2_grep.loader.Loader` + """ + + self.backend = BACKENDS[BACKEND_TYPE]( + {'connection': BACKEND_CONNECTION}) + + def load(self, message: Union[dict, str], topic: str = None) -> None: + """ + Register a notification message + + :param message: `dict` or `str` of notification message + :param topic: `str` of incoming topic (default is `None`) + + :returns: `None` + """ + + if isinstance(message, dict): + LOGGER.debug('Notification message is already a dict') + self.message = message + elif isinstance(message, str): + LOGGER.debug('Notification message is a string; parsing') + try: + self.message = json.loads(message) + except json.decoder.JSONDecodeError as err: + LOGGER.warning(err) + return + + LOGGER.debug('Adding topic to message') + self.message['properties']['topic'] = topic + + LOGGER.debug(f'Notification message: {json.dumps(self.message, indent=4)}') # noqa + + LOGGER.info('Publishing notification message to backend') + self._publish() + + def _publish(self): + """ + Publish notification message from `wis2_grep.loader.Loader.message` + to backend + + :returns: `None` + """ + + LOGGER.info(f'Saving to {BACKEND_TYPE} ({BACKEND_CONNECTION})') + self.backend.save(self.message) + + def __repr__(self): + return '' + + +@click.command() +@click.pass_context +@click.option('--yes', '-y', 'bypass', is_flag=True, default=False, + help='Bypass permission prompts') +@cli_options.OPTION_VERBOSITY +def setup(ctx, bypass, verbosity='NOTSET'): + """Create Global Replay Service backend""" + + if not bypass: + if not click.confirm('Create Global Replay Service backend? This will overwrite existing collections', abort=True): # noqa + return + + backend = BACKENDS[BACKEND_TYPE]({'connection': BACKEND_CONNECTION}) + LOGGER.debug(f'Backend: {backend}') + backend.setup() + + +@click.command() +@click.pass_context +@click.option('--yes', '-y', 'bypass', is_flag=True, default=False, + help='Bypass permission prompts') +@cli_options.OPTION_VERBOSITY +def teardown(ctx, bypass, verbosity='NOTSET'): + """Delete Global Replay Service backend""" + + if not bypass: + if not click.confirm('Delete Global Replay Service backend? This will remove existing collections', abort=True): # noqa + return + + backend = BACKENDS[BACKEND_TYPE]({'connection': BACKEND_CONNECTION}) + LOGGER.debug(f'Backend: {backend}') + backend.teardown() + + +@click.command() +@click.pass_context +@click.argument( + 'path', type=click.Path(exists=True, dir_okay=True, file_okay=True)) +@cli_options.OPTION_VERBOSITY +def load(ctx, path, verbosity='NOTSET'): + """Load notification message""" + + p = Path(path) + + if p.is_file(): + wnms_to_process = [p] + else: + wnms_to_process = p.rglob('*.json') + + for w2p in wnms_to_process: + click.echo(f'Processing {w2p}') + with w2p.open() as fh: + r = Loader() + r.load(fh.read()) diff --git a/wis2-grep.env b/wis2-grep.env new file mode 100644 index 0000000..00a3502 --- /dev/null +++ b/wis2-grep.env @@ -0,0 +1,9 @@ +export WIS2_GREP_LOGGING_LEVEL=DEBUG +export WIS2_GREP_API_URL=http://localhost +export WIS2_GREP_API_URL_DOCKER=http://wis2-grep-api +export WIS2_GREP_BACKEND_TYPE=Elasticsearch +export WIS2_GREP_BACKEND_CONNECTION=http://wis2-grep-backend:9200/wis2-notification-messages +export WIS2_GREP_CENTRE_ID=ca-eccc-msc-global-replay +#export WIS2_GREP_GB=mqtts://everyone:everyone@wis2globalbroker.nws.noaa.gov:8883 +export WIS2_GREP_GB=mqtts://everyone:everyone@globalbroker.meteo.fr:8883 +export WIS2_GREP_GB_TOPIC=origin/a/wis2/#