diff --git a/.github/workflows/pages-deploy.yml b/.github/workflows/pages-deploy.yml new file mode 100644 index 0000000..544ac9d --- /dev/null +++ b/.github/workflows/pages-deploy.yml @@ -0,0 +1,26 @@ +name: Deploy Sphinx documentation to Pages + +on: + push: + tags: + - v*.*.* + +jobs: + pages: + runs-on: ubuntu-20.04 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + permissions: + pages: write + id-token: write + steps: + - uses: actions/checkout@v3 + - name: Update docs configuration with version (tag) name + run: | + sed -i "s/PACKAGE_VERSION/${{ github.ref_name }}/" docs/conf.py + - name: Build and deploy docs + id: deployment + uses: sphinx-notes/pages@v3 + with: + checkout: false \ No newline at end of file diff --git a/CHAP/__init__.py b/CHAP/__init__.py index d699087..131415a 100755 --- a/CHAP/__init__.py +++ b/CHAP/__init__.py @@ -1,3 +1,19 @@ +"""The ChessAnalysisPipeline (CHAP) provides infrastructure to +construct and run X-ray data processing / analysis workflows using a +set of modular components. We call these components `PipelineItem`s +(subclassed into `Reader`s, `Processor`s, and `Writer`s). A `Pipeline` +uses a sequence of `PipelineItem`s to execute a data processing +workflow where the data returned by one `PipelineItem` becomes input +for the next one. + +Many `PipelineItem`s can be shared by data processing workflows for +multiple different X-ray techniques, while others may be unique to +just a single technique. The `PipelineItem`s that are shared by many +techniques are organized in the `CHAP.common` subpackage. +`PipelineItem`s unique to a tomography workflow, for instance, are +organized in the `CHAP.tomo` subpackage. +""" + from CHAP.reader import Reader from CHAP.processor import Processor from CHAP.writer import Writer diff --git a/CHAP/common/__init__.py b/CHAP/common/__init__.py index ed619f8..027a88b 100755 --- a/CHAP/common/__init__.py +++ b/CHAP/common/__init__.py @@ -1,3 +1,15 @@ +"""This subpackage of `CHAP` contains `PipelineItem`s that are or can +be used in workflows for processing data from multiple different X-ray +techniques. + +In addition, `CHAP.common` contains two subpackages of its own: +[`CHAP.common.models`](CHAP.common.models.md) contains tools for +validating input data in some `Processor`s. +[`CHAP.common.utils`](CHAP.common.utils.md) contains a +broad selection of utilities to assist in some common tasks that +appear in specific `Processor` implementations. +""" + from CHAP.common.reader import ( BinaryFileReader, NexusReader, diff --git a/CHAP/common/models/__init__.py b/CHAP/common/models/__init__.py index e69de29..d390727 100755 --- a/CHAP/common/models/__init__.py +++ b/CHAP/common/models/__init__.py @@ -0,0 +1,5 @@ +"""This subpackage contains a collection of +[pydantic](https://docs.pydantic.dev/latest/) models that are used by +`Processors` in [`CHAP.common`](CHAP.common.md) and elsewhere to +validate inputs. +""" diff --git a/CHAP/common/utils/__init__.py b/CHAP/common/utils/__init__.py index e69de29..a062b51 100755 --- a/CHAP/common/utils/__init__.py +++ b/CHAP/common/utils/__init__.py @@ -0,0 +1,4 @@ +"""This subpackage contains generic utilities for fitting, parsing +CHESS scan data, collecting interactive user input, and finding +lattice properties of materials (among others). +""" diff --git a/CHAP/edd/__init__.py b/CHAP/edd/__init__.py index bfb4a57..ab93b00 100755 --- a/CHAP/edd/__init__.py +++ b/CHAP/edd/__init__.py @@ -1,3 +1,6 @@ +"""This subpackage contains `PipelineItems` unique to EDD data +processing workflows. +""" # from CHAP.edd.reader import from CHAP.edd.processor import (DiffractionVolumeLengthProcessor, MCACeriaCalibrationProcessor, diff --git a/CHAP/inference/__init__.py b/CHAP/inference/__init__.py index 4e119f6..c46d3e2 100755 --- a/CHAP/inference/__init__.py +++ b/CHAP/inference/__init__.py @@ -1,3 +1,7 @@ +"""This subpackage contains `PipelineItem`s used to interact with a +[TFaaS inference server](https://github.com/vkuznet/TFaaS/). +""" + # from CHAP.inference.reader import from CHAP.inference.processor import TFaaSImageProcessor # from CHAP.inference.writer import diff --git a/CHAP/processor.py b/CHAP/processor.py index 54d6494..77cba04 100755 --- a/CHAP/processor.py +++ b/CHAP/processor.py @@ -5,6 +5,8 @@ File : processor.py Author : Valentin Kuznetsov Description: Processor module + +Define a generic `Processor` object. """ # system modules @@ -18,11 +20,16 @@ from CHAP.pipeline import PipelineItem class Processor(PipelineItem): - """Processor represent generic processor""" + """Generic data processor. + + The job of any `Processor` in a `Pipeline` is to receive data + returned by the previous `PipelineItem`, process it in some way, + and return the result for the next `PipelineItem` to use as input. + """ def process(self, data): - """Private method to carry out the mechanics of the specific - Processor. + """Extract the contents of the input data, add a string to it, + and return the amended value. :param data: input data :return: processed data @@ -33,7 +40,7 @@ def process(self, data): data = data[0]['data'] if data is None: return [] - # process operation is a simple print function + # process operation is a simple string concatenation data += "process part\n" # and we return data back to pipeline return data diff --git a/CHAP/reader.py b/CHAP/reader.py index 4120feb..ce9c196 100755 --- a/CHAP/reader.py +++ b/CHAP/reader.py @@ -3,6 +3,8 @@ File : reader.py Author : Valentin Kuznetsov Description: generic Reader module + +Define a generic `Reader` object. """ # system modules @@ -17,13 +19,22 @@ class Reader(PipelineItem): - """Reader represent generic file writer""" + """Generic file reader. + + The job of any `Reader` in a `Pipeline` is to provide data stored + in a file to the next `PipelineItem`. Note that a `Reader` used on + its own disrupts the flow of data in a `Pipeline` -- it does not + receive or pass along any data returned by the previous + `PipelineItem`. + """ def read(self, filename): - """Read and return the data from requested from `filename` + """Read and return the contents of `filename` as text :param filename: Name of file to read from - :return: specific number of bytes from a file + :type filename: str + :return: entire contents of the file + :rtype: str """ if not filename: diff --git a/CHAP/saxswaxs/__init__.py b/CHAP/saxswaxs/__init__.py index 471c2e2..bb647b2 100755 --- a/CHAP/saxswaxs/__init__.py +++ b/CHAP/saxswaxs/__init__.py @@ -1,3 +1,6 @@ +"""This subpackage contains `PipelineItems` unique to SAXS/WAXS data +processing workflows. +""" # from CHAP.saxswaxs.reader import # from CHAP.saxswaxs.processor import # from CHAP.saxswaxs.writer import diff --git a/CHAP/sin2psi/__init__.py b/CHAP/sin2psi/__init__.py index b303d90..6d907aa 100755 --- a/CHAP/sin2psi/__init__.py +++ b/CHAP/sin2psi/__init__.py @@ -1,3 +1,7 @@ +"""This subpackage contains `PipelineItems` unique to $\sin^2(\psi)$ +data processing workflows. +""" + # from CHAP.sin2psi.reader import # from CHAP.sin2psi.processor import # from CHAP.sin2psi.writer import diff --git a/CHAP/tomo/__init__.py b/CHAP/tomo/__init__.py index 0f1687c..37f1334 100755 --- a/CHAP/tomo/__init__.py +++ b/CHAP/tomo/__init__.py @@ -1,3 +1,7 @@ +"""This subpackage contains `PipelineItems` unique to tomography data +processing workflows. +""" + from CHAP.common import MapProcessor from CHAP.tomo.processor import ( TomoDataProcessor, diff --git a/CHAP/writer.py b/CHAP/writer.py index bf6f829..aa8eed4 100755 --- a/CHAP/writer.py +++ b/CHAP/writer.py @@ -3,6 +3,8 @@ File : writer.py Author : Valentin Kuznetsov Description: generic Writer module + +Define a generic `Writer` object. """ # system modules @@ -17,9 +19,25 @@ class Writer(PipelineItem): - """Writer represent generic file writer""" + """Generic file writer + + The job of any `Writer` in a `Pipeline` is to receive input + returned by a previous `PipelineItem`, write its data to a + particular file format, then return the same data unaltered so it + can be used by a successive `PipelineItem`. + """ def write(self, data, filename): + """Write the input data as text to a file. + + :param data: input data + :type data: list[PipelineData] + :param filename: Name of file to write to + :type filename: str + :return: contents of the input data + :rtype: object + """ + data = self.unwrap_pipelinedata(data) with open(filename, 'a') as file: file.write(data) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/chap_cli.md b/docs/chap_cli.md new file mode 100644 index 0000000..f9e4123 --- /dev/null +++ b/docs/chap_cli.md @@ -0,0 +1,6 @@ +# CHAP CLI +```{eval-rst} +.. argparse:: + :ref: CHAP.runner.parser + :prog: CHAP +``` \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..638d26f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,45 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'ChessAnalysisPipeline' +copyright = '2023 CHESSComputing' +author = 'V. Kuznetsov, K. Soloway, R. Verberg' +release = 'PACKAGE_VERSION' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + 'myst_parser', + 'autodoc2', + 'sphinxarg.ext' +] +autodoc2_packages = [ + '../CHAP', + '../MLaaS' +] +autodoc2_render_plugin = 'myst' +myst_enable_extensions = ['fieldlist'] + +templates_path = ['_templates'] +exclude_patterns = [] + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_static_path = ['_static'] +html_show_copyright = False +html_theme = 'classic' +html_sidebars = { + '**': ['globaltoc.html', + 'localtoc.html', + 'relations.html', + 'searchbox.html'] +} diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..e83f8b4 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,10 @@ +# ChessAnalysisPipeline (CHAP) + +```{toctree} +:maxdepth: 2 +:caption: Contents +chap_cli.md +apidocs/index.rst +Galaxy.md +ShedTool.md +``` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..387bcd5 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +sphinx +myst-parser +sphinx-autodoc2 +sphinx-argparse