diff --git a/.codacy.yml b/.codacy.yml new file mode 100644 index 0000000..9700d78 --- /dev/null +++ b/.codacy.yml @@ -0,0 +1,8 @@ +--- +engines: + pylint: + enabled: true + python_version: 3 +exclude_paths: + - 'tests/**' + - 'docs/source/conf.py' diff --git a/.cruft.json b/.cruft.json new file mode 100644 index 0000000..b5cd0e3 --- /dev/null +++ b/.cruft.json @@ -0,0 +1,27 @@ +{ + "template": "https://github.com/bird-house/cookiecutter-birdhouse.git", + "commit": "bc8a389e02a3e55e55dad7657671d91e2f238ed9", + "checkout": null, + "context": { + "cookiecutter": { + "full_name": "Carsten Ehbrecht", + "email": "ehbrecht@dkrz.de", + "github_username": "cehbrecht", + "project_name": "ShearWater", + "project_slug": "shearwater", + "project_repo_name": "shearwater", + "project_readthedocs_name": "shearwater", + "project_short_description": "A WPS for forecasting ctropical-cyclone activities.", + "version": "0.1.0", + "open_source_license": "Apache Software License 2.0", + "http_port": "5000", + "use_pytest": "y", + "create_author_file": "y", + "_copy_without_render": [ + "{{cookiecutter.project_slug}}/templates/*.cfg" + ], + "_template": "https://github.com/bird-house/cookiecutter-birdhouse.git" + } + }, + "directory": null +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e0d5f5a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +docs/ +tests/ diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ad431c8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,41 @@ +# http://editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf +charset = utf-8 + +# Docstrings and comments use max_line_length = 79 +[*.py] +max_line_length = 120 + +# Use 2 spaces for the HTML files +[*.html] +indent_size = 2 + +# The JSON files contain newlines inconsistently +[*.json] +indent_size = 2 +insert_final_newline = ignore + +[**/admin/js/vendor/**] +indent_style = ignore +indent_size = ignore + +# Minified JavaScript files shouldn't be changed +[**.min.js] +indent_style = ignore +insert_final_newline = ignore + +# Makefiles always use tabs for indentation +[Makefile] +indent_style = tab + +# Batch files use tabs for indentation +[*.bat] +indent_style = tab diff --git a/.eggs/README.txt b/.eggs/README.txt new file mode 100644 index 0000000..5d01668 --- /dev/null +++ b/.eggs/README.txt @@ -0,0 +1,6 @@ +This directory contains eggs that were downloaded by setuptools to build, test, and run plug-ins. + +This directory caches those eggs to prevent repeated downloads. + +However, it is safe to delete this directory. + diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..6fc50c6 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,6 @@ +# Contributing + +Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. + +Please read the Birdhouse [Developer Guide](https://birdhouse.readthedocs.io/en/latest/dev_guide.html) +and the [ShearWater Documentation](https://shearwater.readthedocs.io/en/latest/) to get started. diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..e6206fd --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,19 @@ +## Description + +Describe what you were trying to get done or your feature request. + +## Environment + +* ShearWater version used, if any: +* Python version, if any: +* Operating System: + +## Steps to Reproduce + +``` +Paste the command(s) you ran and the output. +``` + +## Additional Information + +Links to other issues or sources. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..da62ccd --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ +## Overview + +This PR fixes [issue id] + +Changes: + +* Added ... + +## Related Issue / Discussion + +## Additional Information + +Links to other issues or sources. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..0a0a2f2 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,39 @@ + +name: build ⚙️ + +on: [ push, pull_request ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Install packages + run: | + sudo apt-get -y install pandoc + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install requirements 📦 + run: | + python -m pip install --upgrade pip + pip install -e . + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi + - name: Test with pytest ⚙️ + run: make test + - name: Lint with flake8 ⚙️ + run: make lint + if: matrix.python-version == 3.7 + - name: Build docs 🏗️ + run: make docs + if: false + diff --git a/.gitignore b/.gitignore index 68bc17f..c9c2581 100644 --- a/.gitignore +++ b/.gitignore @@ -1,132 +1,96 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class +# installer +#Makefile -# C extensions -*.so +# Docker +#Dockerfile -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg +# PyWPS +custom.cfg +.custom.cfg +*.pid + +# Python / Extensions etc. +*~ +*.mo +*.so +*.pyc +*.pyo *.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* +*.egg-info +*.sqlite +*.bak +__pycache__ + +# Unit test / Coverage reports .cache +.pytest_cache +.coverage +.tox nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal +unit_tests/testdata.json +coverage/ -# Flask stuff: -instance/ -.webassets-cache +# R +*.Rhistory -# Scrapy stuff: -.scrapy +# Eclipse / PyDev +.project +.pydevproject +.settings -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ +# PyCharm +*.idea + +# Kate +*.kate-swp + +# Sublime Text Editor +*.sublime* + +# buildout +bin +develop-eggs +eggs +parts +build +dist +downloads +.installed.cfg +.mr.developer.cfg +bootstrap-buildout.py +bootstrap.py +#generated by buildout + +*.pid + +# sphinx +#docs/Makefile +docs/make.bat +docs/doctrees/ +docs/html/ +docs/build/ +docs/source/output-sanitize.cfg + +# External Sources +#src/external +src/ + +# tests +*.log +*.lock +testdata.json -# Jupyter Notebook +# IPython .ipynb_checkpoints -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ +# gcc/fortran +*.o +*.a +*.mod +*.out + +# Merge conflict +*.orig # Spyder project settings .spyderproject @@ -140,21 +104,6 @@ venv.bak/ # mypy .mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +# IDE settings +.vscode/ diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..d92d702 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,30 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + # fail_on_warning might generate hard to fix error, in this case it can be + # disabled but this also means those errors will fail silently, choose wisely. + fail_on_warning: true + +# Build documentation with MkDocs +#mkdocs: +# configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +# Optionally set the version of Python and requirements required to build your docs +#python: +# version: 3.6 + +conda: + environment: environment-docs.yml + +build: + image: stable diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2fd6228 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,49 @@ +dist: xenial # required for Python >= 3.7 +language: python +os: + - linux +python: + - "3.7" + - "3.8" + - "3.9" +jobs: + include: + - os: osx + language: shell + - env: ALLOW_FAIL=true + python: "3.8" + allow_failures: + - env: ALLOW_FAIL=true + +before_install: + # Useful for debugging Travis CI environment + - printenv +install: + # Python 3.x is default + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda install setuptools + - conda update -q conda + # Useful for debugging any issues with conda + - conda info -a + # Prepare env with Python version + - conda create -n shearwater -c conda-forge python=$TRAVIS_PYTHON_VERSION + # Update now the env with our environment + - conda env update -f environment.yml + - source activate shearwater + # Packages for testing, generating docs and installing WPS + - make develop +before_script: + # Start WPS service on port 5000 on 0.0.0.0 + - shearwater start --daemon --bind-host 0.0.0.0 --port 5000 + - sleep 2 +script: + - pytest + - make test-notebooks + - flake8 + - make docs # default html + - make SPHINXOPTS='-b epub' docs # to match RtD + - make SPHINXOPTS='-b latex' docs # to match RtD diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..3faa8ed --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,13 @@ +======= +Credits +======= + +Development Lead +---------------- + +* Carsten Ehbrecht + +Contributors +------------ + +None yet. Why not be the first? diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..42b2e4e --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,7 @@ +Changes +******* + +0.1.0 (2023-10-11) +================== + +* First release. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..7a73e88 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,128 @@ +.. highlight:: shell + +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/cehbrecht/shearwater/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +ShearWater could always use more documentation, whether as part of the +official ShearWater docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/cehbrecht/shearwater/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `shearwater` for local development. + +1. Fork the `shearwater` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:your_name_here/shearwater.git + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ mkvirtualenv shearwater + $ cd shearwater/ + $ python setup.py develop + +4. Create a branch for local development:: + + $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8 and the + tests, including testing other Python versions with tox:: + + $ flake8 shearwater tests + $ python setup.py test or pytest + $ tox + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub:: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.5, 3.6, 3.7 and 3.8, and for PyPy. Check + https://travis-ci.com/cehbrecht/shearwater/pull_requests + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + +$ pytest tests.test_shearwater + + +Deploying +--------- + +A reminder for the maintainers on how to deploy. +Make sure all your changes are committed (including an entry in HISTORY.rst). +Then run:: + +$ bump2version patch # possible: major / minor / patch +$ git push +$ git push --tags + +Travis will then deploy to PyPI if tests pass. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a2e0fe3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +# vim:set ft=dockerfile: +FROM continuumio/miniconda3 +MAINTAINER https://github.com/cehbrecht/shearwater +LABEL Description="ShearWater WPS" Vendor="Birdhouse" Version="0.1.0" + +# Update Debian system +RUN apt-get update && apt-get install -y \ + build-essential \ +&& rm -rf /var/lib/apt/lists/* + +# Update conda +RUN conda update -n base conda + +# Copy WPS project +COPY . /opt/wps + +WORKDIR /opt/wps + +# Create conda environment with PyWPS +RUN ["conda", "env", "create", "-n", "wps", "-f", "environment.yml"] + +# Install WPS +RUN ["/bin/bash", "-c", "source activate wps && pip install -e ."] + +# Start WPS service on port 5000 on 0.0.0.0 +EXPOSE 5000 +ENTRYPOINT ["/bin/bash", "-c"] +CMD ["source activate wps && exec shearwater start -b 0.0.0.0 -c /opt/wps/etc/demo.cfg"] + +# docker build -t cehbrecht/shearwater . +# docker run -p 5000:5000 cehbrecht/shearwater +# http://localhost:5000/wps?request=GetCapabilities&service=WPS +# http://localhost:5000/wps?request=DescribeProcess&service=WPS&identifier=all&version=1.0.0 diff --git a/LICENSE b/LICENSE index 261eeb9..b463e1c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,19 @@ +Apache Software License 2.0 + +Copyright (c) 2023, Carsten Ehbrecht + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -173,29 +189,4 @@ incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..5a037a5 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include Makefile +include *.txt +include *.rst +include tox.ini +recursive-include shearwater * +global-exclude __pycache__ +global-exclude *.py[co] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6ead2af --- /dev/null +++ b/Makefile @@ -0,0 +1,158 @@ +# Configuration +APP_ROOT := $(abspath $(lastword $(MAKEFILE_LIST))/..) +APP_NAME := shearwater + +WPS_URL = http://localhost:5000 + +# Used in target refresh-notebooks to make it looks like the notebooks have +# been refreshed from the production server below instead of from the local dev +# instance so the notebooks can also be used as tutorial notebooks. +OUTPUT_URL = https://pavics.ouranos.ca/wpsoutputs + +SANITIZE_FILE := https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/output-sanitize.cfg + +# end of configuration + +.DEFAULT_GOAL := help + +.PHONY: all +all: help + +.PHONY: help +help: + @echo "Please use 'make ' where is one of:" + @echo " help to print this help message. (Default)" + @echo " install to install app by running 'pip install -e .'" + @echo " develop to install with additional development requirements." + @echo " start to start $(APP_NAME) service as daemon (background process)." + @echo " stop to stop $(APP_NAME) service." + @echo " restart to restart $(APP_NAME) service." + @echo " status to show status of $(APP_NAME) service." + @echo " clean to remove all files generated by build and tests." + @echo "\nTesting targets:" + @echo " test to run tests (but skip long running tests)." + @echo " test-all to run all tests (including long running tests)." + @echo " test-notebooks to verify Jupyter Notebook test outputs are valid." + @echo " lint to run code style checks with flake8." + @echo " refresh-notebooks to verify Jupyter Notebook test outputs are valid." + @echo "\nSphinx targets:" + @echo " docs to generate HTML documentation with Sphinx." + @echo "\nDeployment targets:" + @echo " dist to build source and wheel package." + +## Build targets + +.PHONY: install +install: + @echo "Installing application ..." + @-bash -c 'pip install -e .' + @echo "\nStart service with \`make start'" + +.PHONY: develop +develop: + @echo "Installing development requirements for tests and docs ..." + @-bash -c 'pip install -e ".[dev]"' + +.PHONY: start +start: + @echo "Starting application ..." + @-bash -c "$(APP_NAME) start -d" + +.PHONY: stop +stop: + @echo "Stopping application ..." + @-bash -c "$(APP_NAME) stop" + +.PHONY: restart +restart: stop start + @echo "Restarting application ..." + +.PHONY: status +status: + @echo "Showing status ..." + @-bash -c "$(APP_NAME) status" + +.PHONY: clean +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +.PHONY: clean-build +clean-build: + @echo "Removing build artifacts ..." + @-rm -fr build/ + @-rm -fr dist/ + @-rm -fr .eggs/ + @-find . -name '*.egg-info' -exec rm -fr {} + + @-find . -name '*.egg' -exec rm -f {} + + @-find . -name '*.log' -exec rm -fr {} + + @-find . -name '*.sqlite' -exec rm -fr {} + + +.PHONY: clean-pyc +clean-pyc: + @echo "Removing Python file artifacts ..." + @-find . -name '*.pyc' -exec rm -f {} + + @-find . -name '*.pyo' -exec rm -f {} + + @-find . -name '*~' -exec rm -f {} + + @-find . -name '__pycache__' -exec rm -fr {} + + +.PHONY: clean-test +clean-test: + @echo "Removing test artifacts ..." + @-rm -fr .pytest_cache + +.PHONY: clean-dist +clean-dist: clean + @echo "Running 'git clean' ..." + @git diff --quiet HEAD || echo "There are uncommitted changes! Aborting 'git clean' ..." + ## do not use git clean -e/--exclude here, add them to .gitignore instead + @-git clean -dfx + +## Test targets + +.PHONY: test +test: + @echo "Running tests (skip slow and online tests) ..." + @bash -c 'pytest -v -m "not slow and not online" tests/' + +.PHONY: test-all +test-all: + @echo "Running all tests (including slow and online tests) ..." + @bash -c 'pytest -v tests/' + +.PHONY: notebook-sanitizer +notebook-sanitizer: + @echo "Copying notebook output sanitizer ..." + @-bash -c "curl -L $(SANITIZE_FILE) -o $(CURDIR)/docs/source/output-sanitize.cfg --silent" + +.PHONY: test-notebooks +test-notebooks: notebook-sanitizer + @echo "Running notebook-based tests" + @bash -c "env WPS_URL=$(WPS_URL) pytest --nbval --verbose $(CURDIR)/docs/source/notebooks/ --sanitize-with $(CURDIR)/docs/source/output-sanitize.cfg --ignore $(CURDIR)/docs/source/notebooks/.ipynb_checkpoints" + +.PHONY: lint +lint: + @echo "Running flake8 code style checks ..." + @bash -c 'flake8' + +.PHONY: refresh-notebooks +refresh-notebooks: + @echo "Refresh all notebook outputs under docs/source/notebooks" + @bash -c 'for nb in $(CURDIR)/docs/source/notebooks/*.ipynb; do WPS_URL="$(WPS_URL)" jupyter nbconvert --to notebook --execute --ExecutePreprocessor.timeout=60 --output "$$nb" "$$nb"; sed -i "s@$(WPS_URL)/outputs/@$(OUTPUT_URL)/@g" "$$nb"; done; cd $(APP_ROOT)' + +## Sphinx targets + +.PHONY: docs +docs: + @echo "Generating docs with Sphinx ..." + @bash -c '$(MAKE) -C $@ clean html' + @echo "Open your browser to: file:/$(APP_ROOT)/docs/build/html/index.html" + ## do not execute xdg-open automatically since it hangs travis and job does not complete + @echo "xdg-open $(APP_ROOT)/docs/build/html/index.html" + +## Deployment targets + +.PHONY: dist +dist: clean + @echo "Building source and wheel package ..." + @-python setup.py sdist + @-python setup.py bdist_wheel + @-bash -c 'ls -l dist/' diff --git a/README.md b/README.md deleted file mode 100644 index 57e0a3e..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# shearwater -ShearWater diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..6e1922f --- /dev/null +++ b/README.rst @@ -0,0 +1,61 @@ +========== +ShearWater +========== + + +.. image:: https://img.shields.io/pypi/v/shearwater.svg + :target: https://pypi.python.org/pypi/shearwater + +.. image:: https://img.shields.io/travis/cehbrecht/shearwater.svg + :target: https://travis-ci.com/cehbrecht/shearwater + +.. image:: https://readthedocs.org/projects/shearwater/badge/?version=latest + :target: https://shearwater.readthedocs.io/en/latest/?version=latest + :alt: Documentation Status + +.. image:: https://img.shields.io/github/license/cehbrecht/shearwater.svg + :target: https://github.com/cehbrecht/shearwater/blob/master/LICENSE.txt + :alt: GitHub license + +.. image:: https://badges.gitter.im/bird-house/birdhouse.svg + :target: https://gitter.im/bird-house/birdhouse?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge + :alt: Join the chat at https://gitter.im/bird-house/birdhouse + +ShearWater (the bird) + *ShearWater is a bird ...* + +A WPS for forecasting ctropical-cyclone activities. + +Documentation +------------- + +Learn more about ShearWater in its official documentation at +https://shearwater.readthedocs.io. + +Submit bug reports, questions and feature requests at +https://github.com/cehbrecht/shearwater/issues + +Contributing +------------ + +You can find information about contributing in our `Developer Guide`_. + +Please use bumpversion_ to release a new version. + + +License +------- + +* Free software: Apache Software License 2.0 +* Documentation: https://shearwater.readthedocs.io. + + +Credits +------- + +This package was created with Cookiecutter_ and the `bird-house/cookiecutter-birdhouse`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`bird-house/cookiecutter-birdhouse`: https://github.com/bird-house/cookiecutter-birdhouse +.. _`Developer Guide`: https://shearwater.readthedocs.io/en/latest/dev_guide.html +.. _bumpversion: https://shearwater.readthedocs.io/en/latest/dev_guide.html#bump-a-new-version diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..42fa311 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' +services: + wps: + build: . + image: cehbrecht/shearwater + ports: + - "5000:5000" + + +# docker-compose build +# docker-compose up +# docker-compose down +# docker-compose rm diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..977d0c3 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = shearwater +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" -W $(SPHINXOPTS) $(O) diff --git a/docs/source/_static/README.txt b/docs/source/_static/README.txt new file mode 100644 index 0000000..764fc26 --- /dev/null +++ b/docs/source/_static/README.txt @@ -0,0 +1 @@ +Folder for static files diff --git a/docs/source/_static/birdhouse_logo.svg b/docs/source/_static/birdhouse_logo.svg new file mode 100644 index 0000000..1846d23 --- /dev/null +++ b/docs/source/_static/birdhouse_logo.svg @@ -0,0 +1,79 @@ + + + + + + + + + + + diff --git a/docs/source/_static/favicon.ico b/docs/source/_static/favicon.ico new file mode 100644 index 0000000..51386f6 Binary files /dev/null and b/docs/source/_static/favicon.ico differ diff --git a/docs/source/authors.rst b/docs/source/authors.rst new file mode 100644 index 0000000..7739272 --- /dev/null +++ b/docs/source/authors.rst @@ -0,0 +1 @@ +.. include:: ../../AUTHORS.rst diff --git a/docs/source/changes.rst b/docs/source/changes.rst new file mode 100644 index 0000000..d76c92b --- /dev/null +++ b/docs/source/changes.rst @@ -0,0 +1 @@ +.. include:: ../../CHANGES.rst diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100755 index 0000000..529c51e --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +# +# shearwater documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 9 13:47:02 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another +# directory, add these directories to sys.path here. If the directory is +# relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +# +import os +import sys + +# Add shearwater to sys.path to avoid having to full +# install shearwater for autodoc. +# Full install of shearwater will burst memory limit on ReadTheDocs. +sys.path.insert(0, os.path.abspath("../../")) + + +# -- General configuration --------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", + "sphinx.ext.mathjax", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "pywps.ext_autodoc", + "sphinx.ext.autosectionlabel", + "sphinx.ext.imgconverter", + "nbsphinx", + "IPython.sphinxext.ipython_console_highlighting", +] + +# To avoid having to install these and burst memory limit on ReadTheDocs. +# List of all tested working mock imports from all birds so new birds can +# inherit without having to test which work which do not. +autodoc_mock_imports = ["numpy", "xarray", "fiona", "rasterio", "shapely", + "osgeo", "geopandas", "pandas", "statsmodels", + "affine", "rasterstats", "spotpy", "matplotlib", + "scipy", "unidecode", "gdal", "sentry_sdk", "dask", + "numba", "parse", "siphon", "sklearn", "cftime", + "netCDF4", "bottleneck", "ocgis", "geotiff", "geos", + "hdf4", "hdf5", "zlib", "pyproj", "proj", "cartopy", + "scikit-learn", "cairo"] + +# Monkeypatch constant because the following are mock imports. +# Only works if numpy is actually installed and at the same time being mocked. +#import numpy +#numpy.pi = 3.1416 + +# We are using mock imports in readthedocs, so probably safer to not run the notebooks +nbsphinx_execute = 'never' + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "ShearWater" +copyright = "2023, Carsten Ehbrecht" +author = "Carsten Ehbrecht" + +# The version info for the project you're documenting, acts as replacement +# for |version| and |release|, also used in various other places throughout +# the built documents. +# +# The short X.Y version. +version = shearwater.__version__ +# The full version, including alpha/beta/rc tags. +release = shearwater.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# Suppress "WARNING: unknown mimetype for ..." when building EPUB. +suppress_warnings = ['epub.unknown_project_files'] + +# Avoid "configuration.rst:4:duplicate label configuration, other instance in configuration.rst" +autosectionlabel_prefix_document = True + + +# -- Options for HTML output ------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a +# theme further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +html_logo = "_static/birdhouse_logo.svg" + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = "_static/favicon.ico" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + + +# -- Options for HTMLHelp output --------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "shearwaterdoc" + + +# -- Options for LaTeX output ------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "shearwater.tex", + "ShearWater Documentation", + "Carsten Ehbrecht", + "manual", + ), +] + + +# -- Options for manual page output ------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + master_doc, + "shearwater", + "ShearWater Documentation", + [author], + 1, + ) +] + + +# -- Options for Texinfo output ---------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "shearwater", + "ShearWater Documentation", + author, + "shearwater", + "A WPS for forecasting ctropical-cyclone activities.", + "Miscellaneous", + ), +] diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst new file mode 100644 index 0000000..ce11079 --- /dev/null +++ b/docs/source/configuration.rst @@ -0,0 +1,49 @@ +.. _configuration: + +Configuration +============= + +Command-line options +-------------------- + +You can overwrite the default `PyWPS`_ configuration by using command-line options. +See the ShearWater help which options are available:: + + $ shearwater start --help + --hostname HOSTNAME hostname in PyWPS configuration. + --port PORT port in PyWPS configuration. + +Start service with different hostname and port:: + + $ shearwater start --hostname localhost --port 5001 + +Use a custom configuration file +------------------------------- + +You can overwrite the default `PyWPS`_ configuration by providing your own +PyWPS configuration file (just modifiy the options you want to change). +Use one of the existing ``sample-*.cfg`` files as example and copy them to ``etc/custom.cfg``. + +For example change the hostname (*demo.org*) and logging level: + +.. code-block:: console + + $ cd shearwater + $ vim etc/custom.cfg + $ cat etc/custom.cfg + [server] + url = http://demo.org:5000/wps + outputurl = http://demo.org:5000/outputs + + [logging] + level = DEBUG + +Start the service with your custom configuration: + +.. code-block:: console + + # start the service with this configuration + $ shearwater start -c etc/custom.cfg + + +.. _PyWPS: http://pywps.org/ diff --git a/docs/source/dev_guide.rst b/docs/source/dev_guide.rst new file mode 100644 index 0000000..a80f1d3 --- /dev/null +++ b/docs/source/dev_guide.rst @@ -0,0 +1,107 @@ +.. _devguide: + +Developer Guide +=============== + +.. contents:: + :local: + :depth: 1 + +.. WARNING:: To create new processes look at examples in Emu_. + +Building the docs +----------------- + +First install dependencies for the documentation: + +.. code-block:: console + + $ make develop + +Run the Sphinx docs generator: + +.. code-block:: console + + $ make docs + +.. _testing: + +Running tests +------------- + +Run tests using pytest_. + +First activate the ``shearwater`` Conda environment and install ``pytest``. + +.. code-block:: console + + $ source activate shearwater + $ pip install -r requirements_dev.txt # if not already installed + OR + $ make develop + +Run quick tests (skip slow and online): + +.. code-block:: console + + $ pytest -m 'not slow and not online'" + +Run all tests: + +.. code-block:: console + + $ pytest + +Check pep8: + +.. code-block:: console + + $ flake8 + +Run tests the lazy way +---------------------- + +Do the same as above using the ``Makefile``. + +.. code-block:: console + + $ make test + $ make test-all + $ make lint + +Prepare a release +----------------- + +Update the Conda specification file to build identical environments_ on a specific OS. + +.. note:: You should run this on your target OS, in our case Linux. + +.. code-block:: console + + $ conda env create -f environment.yml + $ source activate shearwater + $ make clean + $ make install + $ conda list -n shearwater --explicit > spec-file.txt + +.. _environments: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments + + +Bump a new version +------------------ + +Make a new version of ShearWater in the following steps: + +* Make sure everything is commit to GitHub. +* Update ``CHANGES.rst`` with the next version. +* Dry Run: ``bumpversion --dry-run --verbose --new-version 0.8.1 patch`` +* Do it: ``bumpversion --new-version 0.8.1 patch`` +* ... or: ``bumpversion --new-version 0.9.0 minor`` +* Push it: ``git push`` +* Push tag: ``git push --tags`` + +See the bumpversion_ documentation for details. + +.. _bumpversion: https://pypi.org/project/bumpversion/ +.. _pytest: https://docs.pytest.org/en/latest/ +.. _Emu: https://github.com/bird-house/emu diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..318de0c --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,19 @@ +.. include:: ../../README.rst + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + installation + configuration + notebooks/index + dev_guide + processes + authors + changes + +Indices and tables +================== +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 0000000..82f2548 --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,113 @@ +.. _installation: + +Installation +============ + +.. contents:: + :local: + :depth: 1 + +Install from Conda +------------------ + +.. warning:: + + TODO: Prepare Conda package. + +Install from GitHub +------------------- + +Check out code from the ShearWater GitHub repo and start the installation: + +.. code-block:: console + + $ git clone https://github.com/cehbrecht/shearwater.git + $ cd shearwater + +Create Conda environment named `shearwater`: + +.. code-block:: console + + $ conda env create -f environment.yml + $ source activate shearwater + +Install ShearWater app: + +.. code-block:: console + + $ pip install -e . + OR + make install + +For development you can use this command: + +.. code-block:: console + + $ pip install -e .[dev] + OR + $ make develop + +Start ShearWater PyWPS service +------------------------------ + +After successful installation you can start the service using the ``shearwater`` command-line. + +.. code-block:: console + + $ shearwater --help # show help + $ shearwater start # start service with default configuration + + OR + + $ shearwater start --daemon # start service as daemon + loading configuration + forked process id: 42 + +The deployed WPS service is by default available on: + +http://localhost:5000/wps?service=WPS&version=1.0.0&request=GetCapabilities. + +.. NOTE:: Remember the process ID (PID) so you can stop the service with ``kill PID``. + +You can find which process uses a given port using the following command (here for port 5000): + +.. code-block:: console + + $ netstat -nlp | grep :5000 + + +Check the log files for errors: + +.. code-block:: console + + $ tail -f pywps.log + +... or do it the lazy way ++++++++++++++++++++++++++ + +You can also use the ``Makefile`` to start and stop the service: + +.. code-block:: console + + $ make start + $ make status + $ tail -f pywps.log + $ make stop + + +Run ShearWater as Docker container +---------------------------------- + +You can also run ShearWater as a Docker container. + +.. warning:: + + TODO: Describe Docker container support. + +Use Ansible to deploy ShearWater on your System +----------------------------------------------- + +Use the `Ansible playbook`_ for PyWPS to deploy ShearWater on your system. + + +.. _Ansible playbook: http://ansible-wps-playbook.readthedocs.io/en/latest/index.html diff --git a/docs/source/notebooks/example.ipynb b/docs/source/notebooks/example.ipynb new file mode 100644 index 0000000..56e8305 --- /dev/null +++ b/docs/source/notebooks/example.ipynb @@ -0,0 +1,48 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Usage Example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import shearwater" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/source/notebooks/index.rst b/docs/source/notebooks/index.rst new file mode 100644 index 0000000..cdbc518 --- /dev/null +++ b/docs/source/notebooks/index.rst @@ -0,0 +1,8 @@ +======== +Examples +======== + +.. toctree:: + :maxdepth: 1 + + example diff --git a/docs/source/processes.rst b/docs/source/processes.rst new file mode 100644 index 0000000..f0b1600 --- /dev/null +++ b/docs/source/processes.rst @@ -0,0 +1,15 @@ +.. _processes: + +Processes +========= + +.. contents:: + :local: + :depth: 1 + +Say Hello +--------- + +.. autoprocess:: shearwater.processes.wps_say_hello.SayHello + :docstring: + :skiplines: 1 diff --git a/environment-docs.yml b/environment-docs.yml new file mode 100644 index 0000000..a30796f --- /dev/null +++ b/environment-docs.yml @@ -0,0 +1,10 @@ +# conda env create -f environment-docs.yml +name: shearwater-docs +channels: +- conda-forge +- defaults +dependencies: +- pywps>=4.4 +- sphinx +- nbsphinx +- ipython diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..dd8d389 --- /dev/null +++ b/environment.yml @@ -0,0 +1,13 @@ +name: shearwater +channels: +- conda-forge +- defaults +dependencies: +- pip +- python>=3.7,<3.10 +- pywps>=4.5.1,<4.6 +- jinja2 +- click +- psutil +# tests +- pytest diff --git a/etc/debug.cfg b/etc/debug.cfg new file mode 100644 index 0000000..c00a2b7 --- /dev/null +++ b/etc/debug.cfg @@ -0,0 +1,2 @@ +[logging] +level = DEBUG diff --git a/etc/demo.cfg b/etc/demo.cfg new file mode 100644 index 0000000..52145a3 --- /dev/null +++ b/etc/demo.cfg @@ -0,0 +1,2 @@ +[logging] +level = WARN diff --git a/etc/sample-custom.cfg b/etc/sample-custom.cfg new file mode 100644 index 0000000..7b80725 --- /dev/null +++ b/etc/sample-custom.cfg @@ -0,0 +1,6 @@ +[server] +url = http://demo.org:5000/wps +outputurl = http://demo.org:5000/outputs + +[logging] +level = DEBUG diff --git a/etc/sample-postgres.cfg b/etc/sample-postgres.cfg new file mode 100644 index 0000000..a16b904 --- /dev/null +++ b/etc/sample-postgres.cfg @@ -0,0 +1,3 @@ +[logging] +level = INFO +database = postgresql+psycopg2://postgres:postgres@localhost:5432/postgres diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..01cdf08 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +click +jinja2 +psutil +pywps>=4.5.1,<4.6 diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..878bba6 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,16 @@ +pytest>=6.0 +pytest-runner +pytest-cov +flake8 +tox +pytest-flake8 +ipython +pytest-notebook +nbsphinx +nbval>=0.9.6 +nbconvert +sphinx>=1.8.5 +bump2version +Click +cruft +# Changing dependencies above this comment will create merge conflicts when updating the cookiecutter template with cruft. Add extra requirements below this line. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ada64ab --- /dev/null +++ b/setup.cfg @@ -0,0 +1,50 @@ +[bumpversion] +current_version = 0.1.0 +commit = True +tag = True + +[metadata] +description-file = README.rst + +[bumpversion:file:shearwater/__version__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bumpversion:file:docs/source/conf.py] +parse = version|release = {current_version} +replace = {new_version} + +[bumpversion:file:Dockerfile] +search = Version="{current_version}" +replace = Version="{new_version}" + +[bumpversion:file:.cruft.json] +search = "version": "{current_version}", +replace = "version": "{new_version}", + +[bdist_wheel] +universal = 1 + +[flake8] +max-line-length = 120 +exclude = + .git, + __pycache__, + docs/source/conf.py, + build, + dist, + src, + +[aliases] +# Define setup.py command aliases here +test = pytest + +[tool:pytest] +addopts = + --strict-markers + --tb=native + tests/ +python_files = test_*.py +markers = + online: mark test to need internet connection + slow: mark test to be slow diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..aa934ce --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +"""The setup script.""" + +import os + +from setuptools import setup, find_packages + +here = os.path.abspath(os.path.dirname(__file__)) +README = open(os.path.join(here, 'README.rst')).read() +CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() +REQUIRES_PYTHON = ">=3.6.0" + +about = {} +with open(os.path.join(here, 'shearwater', '__version__.py'), 'r') as f: + exec(f.read(), about) + +setup_requirements = ['pytest-runner', ] + +test_requirements = ['pytest>=3', ] + +requirements = [line.strip() for line in open('requirements.txt')] + +dev_reqs = [line.strip() for line in open('requirements_dev.txt')] + +classifiers = [ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX', + 'Programming Language :: Python', + 'Natural Language :: English', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Scientific/Engineering :: Atmospheric Science', + 'License :: OSI Approved :: Apache Software License', +] + +setup(name='shearwater', + version=about['__version__'], + description="A WPS for forecasting ctropical-cyclone activities.", + long_description=README + '\n\n' + CHANGES, + long_description_content_type="text/x-rst", + author=about['__author__'], + author_email=about['__email__'], + url='https://github.com/cehbrecht/shearwater', + python_requires=REQUIRES_PYTHON, + classifiers=classifiers, + license="Apache Software License 2.0", + keywords='wps pywps birdhouse shearwater', + packages=find_packages(), + include_package_data=True, + install_requires=requirements, + setup_requires=setup_requirements, + test_suite='tests', + tests_require=test_requirements, + extras_require={ + "dev": dev_reqs, # pip install ".[dev]" + }, + entry_points={ + 'console_scripts': [ + 'shearwater=shearwater.cli:cli', + ]},) diff --git a/shearwater/__init__.py b/shearwater/__init__.py new file mode 100644 index 0000000..8e0d2ce --- /dev/null +++ b/shearwater/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +"""Top-level package for ShearWater.""" + +from .__version__ import __author__, __email__, __version__ # noqa: F401 + +from .wsgi import application # noqa: F401 diff --git a/shearwater/__version__.py b/shearwater/__version__.py new file mode 100644 index 0000000..e4a49d3 --- /dev/null +++ b/shearwater/__version__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- + +# This information is located in its own file so that it can be loaded +# without importing the main package when its dependencies are not installed. +# See: https://packaging.python.org/guides/single-sourcing-package-version + +__author__ = "Carsten Ehbrecht" +__email__ = "ehbrecht@dkrz.de" +__version__ = "0.1.0" diff --git a/shearwater/cli.py b/shearwater/cli.py new file mode 100644 index 0000000..bd0aa66 --- /dev/null +++ b/shearwater/cli.py @@ -0,0 +1,184 @@ +########################################################### +# Demo WPS service for testing and debugging. +# +# See the werkzeug documentation on how to use the debugger: +# http://werkzeug.pocoo.org/docs/0.12/debug/ +########################################################### + +import os +import psutil +import click +from jinja2 import Environment, PackageLoader +from pywps import configuration + +from . import wsgi +from urllib.parse import urlparse + +PID_FILE = os.path.abspath(os.path.join(os.path.curdir, "pywps.pid")) + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + +template_env = Environment( + loader=PackageLoader('shearwater', 'templates'), + autoescape=True +) + + +def write_user_config(**kwargs): + config_templ = template_env.get_template('pywps.cfg') + rendered_config = config_templ.render(**kwargs) + config_file = os.path.abspath(os.path.join(os.path.curdir, ".custom.cfg")) + with open(config_file, 'w') as fp: + fp.write(rendered_config) + return config_file + + +def get_host(): + url = configuration.get_config_value('server', 'url') + url = url or 'http://localhost:5000/wps' + + click.echo("starting WPS service on {}".format(url)) + + parsed_url = urlparse(url) + if ':' in parsed_url.netloc: + host, port = parsed_url.netloc.split(':') + port = int(port) + else: + host = parsed_url.netloc + port = 80 + return host, port + + +def run_process_action(action=None): + """Run an action with psutil on current process + and return a status message.""" + action = action or 'status' + try: + with open(PID_FILE, 'r') as fp: + pid = int(fp.read()) + p = psutil.Process(pid) + if action == 'stop': + p.terminate() + msg = "pid={}, status=terminated".format(p.pid) + else: + from psutil import _pprint_secs + msg = "pid={}, status={}, created={}".format( + p.pid, p.status(), _pprint_secs(p.create_time())) + if action == 'stop': + os.remove(PID_FILE) + except IOError: + msg = 'No PID file found. Service not running? Try "netstat -nlp | grep :5000".' + except psutil.NoSuchProcess as e: + msg = e.msg + click.echo(msg) + + +def _run(application, bind_host=None, daemon=False): + from werkzeug.serving import run_simple + # call this *after* app is initialized ... needs pywps config. + host, port = get_host() + bind_host = bind_host or host + # need to serve the wps outputs + static_files = { + '/outputs': configuration.get_config_value('server', 'outputpath') + } + run_simple( + hostname=bind_host, + port=port, + application=application, + use_debugger=False, + use_reloader=False, + threaded=True, + # processes=2, + use_evalex=not daemon, + static_files=static_files) + + +@click.group(context_settings=CONTEXT_SETTINGS) +@click.version_option() +def cli(): + """Command line to start/stop a PyWPS service. + + Do not use this service in a production environment. + It's intended to be running in a test environment only! + For more documentation, visit http://pywps.org/doc + """ + pass + + +@cli.command() +def status(): + """Show status of PyWPS service""" + run_process_action(action='status') + + +@cli.command() +def stop(): + """Stop PyWPS service""" + run_process_action(action='stop') + + +@cli.command() +@click.option('--config', '-c', metavar='PATH', help='path to pywps configuration file.') +@click.option('--bind-host', '-b', metavar='IP-ADDRESS', default='127.0.0.1', + help='IP address used to bind service.') +@click.option('--daemon', '-d', is_flag=True, help='run in daemon mode.') +@click.option('--hostname', metavar='HOSTNAME', default='localhost', help='hostname in PyWPS configuration.') +@click.option('--port', metavar='PORT', default='5000', help='port in PyWPS configuration.') +@click.option('--maxsingleinputsize', default='200mb', help='maxsingleinputsize in PyWPS configuration.') +@click.option('--maxprocesses', metavar='INT', default='10', help='maxprocesses in PyWPS configuration.') +@click.option('--parallelprocesses', metavar='INT', default='2', help='parallelprocesses in PyWPS configuration.') +@click.option('--log-level', metavar='LEVEL', default='INFO', help='log level in PyWPS configuration.') +@click.option('--log-file', metavar='PATH', default='pywps.log', help='log file in PyWPS configuration.') +@click.option('--database', default='sqlite:///pywps-logs.sqlite', help='database in PyWPS configuration') +@click.option('--outputurl', default='', help='base URL for file downloads') +@click.option('--outputpath', default='', help='base directory where outputs are written') +def start(config, bind_host, daemon, hostname, port, + maxsingleinputsize, maxprocesses, parallelprocesses, + log_level, log_file, database, outputurl, outputpath): + """Start PyWPS service. + This service is by default available at http://localhost:5000/wps + """ + if os.path.exists(PID_FILE): + click.echo('PID file exists: "{}". Service still running?'.format(PID_FILE)) + os._exit(0) + cfgfiles = [] + cfgfiles.append(write_user_config( + wps_hostname=hostname, + wps_port=port, + wps_maxsingleinputsize=maxsingleinputsize, + wps_maxprocesses=maxprocesses, + wps_parallelprocesses=parallelprocesses, + wps_log_level=log_level, + wps_log_file=log_file, + wps_database=database, + wps_outputurl=outputurl, + wps_outputpath=outputpath + )) + if config: + cfgfiles.append(config) + app = wsgi.create_app(cfgfiles) + # let's start the service ... + # See: + # * https://github.com/geopython/pywps-flask/blob/master/demo.py + # * http://werkzeug.pocoo.org/docs/0.14/serving/ + if daemon: + # daemon (fork) mode + pid = None + try: + pid = os.fork() + if pid: + click.echo('forked process id: {}'.format(pid)) + with open(PID_FILE, 'w') as fp: + fp.write("{}".format(pid)) + except OSError as e: + raise Exception("%s [%d]" % (e.strerror, e.errno)) + + if pid == 0: + os.setsid() + _run(app, bind_host=bind_host, daemon=True) + else: + os._exit(0) + else: + # no daemon + _run(app, bind_host=bind_host) diff --git a/shearwater/default.cfg b/shearwater/default.cfg new file mode 100644 index 0000000..135c500 --- /dev/null +++ b/shearwater/default.cfg @@ -0,0 +1,20 @@ +[metadata:main] +identification_title = ShearWater +identification_abstract = A WPS for forecasting ctropical-cyclone activities. +identification_keywords = PyWPS, WPS, OGC, processing, birdhouse, shearwater, demo +identification_keywords_type = theme +provider_name = ShearWater +provider_url=https://shearwater.readthedocs.org/en/latest/ + +[server] +url = http://localhost:5000/wps +outputurl = http://localhost:5000/outputs +allowedinputpaths = / +maxsingleinputsize = 200mb +maxprocesses = 10 +parallelprocesses = 2 + +[logging] +level = INFO +file = shearwater.log +format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s diff --git a/shearwater/processes/__init__.py b/shearwater/processes/__init__.py new file mode 100644 index 0000000..437441e --- /dev/null +++ b/shearwater/processes/__init__.py @@ -0,0 +1,5 @@ +from .wps_say_hello import SayHello + +processes = [ + SayHello(), +] diff --git a/shearwater/processes/wps_say_hello.py b/shearwater/processes/wps_say_hello.py new file mode 100644 index 0000000..2d67782 --- /dev/null +++ b/shearwater/processes/wps_say_hello.py @@ -0,0 +1,47 @@ +from pywps import Process, LiteralInput, LiteralOutput, UOM +from pywps.app.Common import Metadata + +import logging +LOGGER = logging.getLogger("PYWPS") + + +class SayHello(Process): + """A nice process saying 'hello'.""" + def __init__(self): + inputs = [ + LiteralInput('name', 'Your name', + abstract='Please enter your name.', + keywords=['name', 'firstname'], + data_type='string')] + outputs = [ + LiteralOutput('output', 'Output response', + abstract='A friendly Hello from us.', + keywords=['output', 'result', 'response'], + data_type='string')] + + super(SayHello, self).__init__( + self._handler, + identifier='hello', + title='Say Hello', + abstract='Just says a friendly Hello.' + 'Returns a literal string output with Hello plus the inputed name.', + keywords=['hello', 'demo'], + metadata=[ + Metadata('PyWPS', 'https://pywps.org/'), + Metadata('Birdhouse', 'http://bird-house.github.io/'), + Metadata('PyWPS Demo', 'https://pywps-demo.readthedocs.io/en/latest/'), + Metadata('Emu: PyWPS examples', 'https://emu.readthedocs.io/en/latest/'), + ], + version='1.5', + inputs=inputs, + outputs=outputs, + store_supported=True, + status_supported=True + ) + + @staticmethod + def _handler(request, response): + LOGGER.info("say hello") + response.outputs['output'].data = 'Hello ' + request.inputs['name'][0].data + response.outputs['output'].uom = UOM('unity') + return response diff --git a/shearwater/templates/pywps.cfg b/shearwater/templates/pywps.cfg new file mode 100644 index 0000000..20951f8 --- /dev/null +++ b/shearwater/templates/pywps.cfg @@ -0,0 +1,27 @@ +[server] +{% if wps_url %} +url = {{ wps_url }} +{% else %} +url = http://{{ wps_hostname }}:{{ wps_port }}/wps +{% endif %} +{% if wps_outputurl %} +outputurl = {{ wps_outputurl }} +{% else %} +outputurl = http://{{ wps_hostname }}:{{ wps_port }}/outputs +{% endif %} +allowedinputpaths = / +maxsingleinputsize = {{ wps_maxsingleinputsize|default('200mb') }} +maxprocesses = {{ wps_maxprocesses|default('10') }} +parallelprocesses = {{ wps_parallelprocesses|default('2') }} +{% if wps_outputpath %} +outputpath= {{ wps_outputpath }} +{% endif %} +{% if wps_workdir %} +workdir={{ wps_workdir }} +{% endif %} + +[logging] +level = {{ wps_log_level|default('INFO') }} +file = {{ wps_log_file|default('pywps.log') }} +database = {{ wps_database|default('sqlite:///pywps-logs.sqlite') }} +format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s diff --git a/shearwater/wsgi.py b/shearwater/wsgi.py new file mode 100644 index 0000000..b9812c3 --- /dev/null +++ b/shearwater/wsgi.py @@ -0,0 +1,17 @@ +import os +from pywps.app.Service import Service + +from .processes import processes + + +def create_app(cfgfiles=None): + config_files = [os.path.join(os.path.dirname(__file__), 'default.cfg')] + if cfgfiles: + config_files.extend(cfgfiles) + if 'PYWPS_CFG' in os.environ: + config_files.append(os.environ['PYWPS_CFG']) + service = Service(processes=processes, cfgfiles=config_files) + return service + + +application = create_app() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..d322c3d --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Unit test package for shearwater.""" diff --git a/tests/common.py b/tests/common.py new file mode 100644 index 0000000..4aa035d --- /dev/null +++ b/tests/common.py @@ -0,0 +1,43 @@ +from pywps import get_ElementMakerForVersion +from pywps.app.basic import get_xpath_ns +from pywps.tests import WpsClient, WpsTestResponse + +VERSION = "1.0.0" +WPS, OWS = get_ElementMakerForVersion(VERSION) +xpath_ns = get_xpath_ns(VERSION) + + +class WpsTestClient(WpsClient): + + def get(self, *args, **kwargs): + query = "?" + for key, value in kwargs.items(): + query += "{0}={1}&".format(key, value) + return super(WpsTestClient, self).get(query) + + +def client_for(service): + return WpsTestClient(service, WpsTestResponse) + + +def get_output(doc): + """Copied from pywps/tests/test_execute.py. + TODO: make this helper method public in pywps.""" + output = {} + for output_el in xpath_ns(doc, '/wps:ExecuteResponse' + '/wps:ProcessOutputs/wps:Output'): + [identifier_el] = xpath_ns(output_el, './ows:Identifier') + + lit_el = xpath_ns(output_el, './wps:Data/wps:LiteralData') + if lit_el != []: + output[identifier_el.text] = lit_el[0].text + + ref_el = xpath_ns(output_el, './wps:Reference') + if ref_el != []: + output[identifier_el.text] = ref_el[0].attrib['href'] + + data_el = xpath_ns(output_el, './wps:Data/wps:ComplexData') + if data_el != []: + output[identifier_el.text] = data_el[0].text + + return output diff --git a/tests/test_shearwater.py b/tests/test_shearwater.py new file mode 100644 index 0000000..c3b2c77 --- /dev/null +++ b/tests/test_shearwater.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +"""Tests for `shearwater` package.""" + +import pytest + +# from click.testing import CliRunner + +# import shearwater +# from shearwater import cli + + +@pytest.fixture +def response(): + """Sample pytest fixture. + + See more at: http://doc.pytest.org/en/latest/fixture.html + """ + # import requests + # return requests.get('https://github.com/audreyr/cookiecutter-pypackage') + + +def test_content(response): + """Sample pytest test function with the pytest fixture as an argument.""" + # from bs4 import BeautifulSoup + # assert 'GitHub' in BeautifulSoup(response.content).title.string diff --git a/tests/test_wps_caps.py b/tests/test_wps_caps.py new file mode 100644 index 0000000..cedde26 --- /dev/null +++ b/tests/test_wps_caps.py @@ -0,0 +1,16 @@ +from pywps import Service + +from .common import client_for +from shearwater.processes import processes + + +def test_wps_caps(): + client = client_for(Service(processes=processes)) + resp = client.get(service='wps', request='getcapabilities', version='1.0.0') + names = resp.xpath_text('/wps:Capabilities' + '/wps:ProcessOfferings' + '/wps:Process' + '/ows:Identifier') + assert sorted(names.split()) == [ + 'hello', + ] diff --git a/tests/test_wps_hello.py b/tests/test_wps_hello.py new file mode 100644 index 0000000..f699323 --- /dev/null +++ b/tests/test_wps_hello.py @@ -0,0 +1,15 @@ +from pywps import Service +from pywps.tests import client_for, assert_response_success + +from .common import get_output +from shearwater.processes.wps_say_hello import SayHello + + +def test_wps_hello(): + client = client_for(Service(processes=[SayHello()])) + datainputs = "name=LovelySugarBird" + resp = client.get( + "?service=WPS&request=Execute&version=1.0.0&identifier=hello&datainputs={}".format( + datainputs)) + assert_response_success(resp) + assert get_output(resp.xml) == {'output': "Hello LovelySugarBird"} diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..2d1548e --- /dev/null +++ b/tox.ini @@ -0,0 +1,29 @@ +[tox] +envlist = py36, py37, py38, flake8 +requires = pip >= 20.0 +opts = -v + +[travis] +python = + 3.8: py38 + 3.7: py37 + 3.6: py36 + +[testenv:flake8] +basepython = python +deps = flake8 +commands = flake8 shearwater tests + +[testenv] +setenv = + PYTHONPATH = {toxinidir} +install_command = python -m pip install --no-user {opts} {packages} +download = True +deps = + -r{toxinidir}/requirements_dev.txt +; If you want to make tox run the tests with the same versions, create a +; requirements.txt with the pinned versions and uncomment the following line: +; -r{toxinidir}/requirements.txt +commands = + pytest --basetemp={envtmpdir} +