diff --git a/.github/workflows/markdown-link-check.yml b/.github/workflows/markdown-link-check.yml index 3cdbce01..301cf731 100644 --- a/.github/workflows/markdown-link-check.yml +++ b/.github/workflows/markdown-link-check.yml @@ -12,9 +12,9 @@ jobs: markdown-link-check: name: Check markdown links - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 # there is an issue with ubuntu-latest and linkspector steps: - - uses: actions/checkout@v3 - - uses: gaurav-nelson/github-action-markdown-link-check@v1 + - uses: actions/checkout@v4 + - uses: umbrelladocs/action-linkspector@v1 with: - config-file: '.mlc-config.json' + config_file: '.linkspector.yml' diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml new file mode 100644 index 00000000..4bc7df01 --- /dev/null +++ b/.github/workflows/publish-container.yml @@ -0,0 +1,59 @@ +# +name: Create and publish the PyStemmusScope container image 📦 + +on: + release: + types: [published] + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index a0a0ed8c..dc3cb65c 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,6 +11,7 @@ name: Upload Python Package on: release: types: [published] + workflow_dispatch: permissions: contents: read diff --git a/.linkspector.yml b/.linkspector.yml new file mode 100644 index 00000000..ce488623 --- /dev/null +++ b/.linkspector.yml @@ -0,0 +1,14 @@ +dirs: + - ./docs +excludedDirs: + - ./docs/notebooks + - ./docs/overrides +ignorePatterns: + - pattern: "^http://localhost" + - pattern: "^https://doi.org/" + - pattern: "^https://github.com/.*/settings/secrets/actions$" + - pattern: "^https://github.com/organizations/.*/repositories/new" + - pattern: "^https://test.pypi.org" + - pattern: "^https://bestpractices.coreinfrastructure.org/projects/" + - pattern: "^https://readthedocs.org/dashboard/import.*" +useGitIgnore: true diff --git a/.mlc-config.json b/.mlc-config.json deleted file mode 100644 index 1d388679..00000000 --- a/.mlc-config.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "_comment": "Markdown Link Checker configuration, see https://github.com/gaurav-nelson/github-action-markdown-link-check and https://github.com/tcort/markdown-link-check", - "ignorePatterns": [ - { - "pattern": "^http://localhost" - }, - { - "pattern": "^https://doi.org/" - }, - { - "pattern": "^https://github.com/.*/settings/secrets/actions$" - }, - { - "pattern": "^https://github.com/organizations/.*/repositories/new" - }, - { - "pattern": "^https://test.pypi.org" - }, - { - "pattern": "^https://bestpractices.coreinfrastructure.org/projects/" - }, - { - "pattern": "^https://readthedocs.org/dashboard/import.*" - } - ], - "replacementPatterns": [ - ], - "retryOn429": true, - "timeout": "20s" -} diff --git a/PyStemmusScope/bmi/__init__.py b/PyStemmusScope/bmi/__init__.py new file mode 100644 index 00000000..80c00e2b --- /dev/null +++ b/PyStemmusScope/bmi/__init__.py @@ -0,0 +1,5 @@ +"""Documentation about StemmusScopeBmi.""" +from .implementation import StemmusScopeBmi + + +__all__ = ["StemmusScopeBmi"] diff --git a/PyStemmusScope/bmi/local_process.py b/PyStemmusScope/bmi/local_process.py index 9d35052e..e4997212 100644 --- a/PyStemmusScope/bmi/local_process.py +++ b/PyStemmusScope/bmi/local_process.py @@ -104,6 +104,7 @@ def __init__(self, cfg_file: str) -> None: env = { "LD_LIBRARY_PATH": lib_path, "MATLAB_LOG_DIR": str(config["OutputPath"]), + "MCR_CACHE_ROOT": str(os.getenv("MCR_CACHE_ROOT")), } self.process = subprocess.Popen( diff --git a/README.md b/README.md index 62e4385d..e1653da5 100644 --- a/README.md +++ b/README.md @@ -18,18 +18,11 @@ [![cffconvert](https://github.com/EcoExtreML/stemmus_scope_processing/actions/workflows/cffconvert.yml/badge.svg)](https://github.com/EcoExtreML/stemmus_scope_processing/actions/workflows/cffconvert.yml) [![markdown-link-check](https://github.com/EcoExtreML/stemmus_scope_processing/actions/workflows/markdown-link-check.yml/badge.svg)](https://github.com/EcoExtreML/stemmus_scope_processing/actions/workflows/markdown-link-check.yml) --> -This repository includes the python package `PyStemmusScope` for running the -STEMMUS-SCOPE model. +This repository includes the python package `PyStemmusScope` for preparing data +nd running the STEMMUS-SCOPE model. The model source code, executable file and utility files are available in the [STEMMUS_SCOPE repository](https://github.com/EcoExtreML/STEMMUS_SCOPE). -The input datasets are available on Snellius and CRIB. First, make sure you have -right access to the repository and data. Then, see the notebook -[run_model_on_different_infra.ipynb](./docs/notebooks/run_model_on_different_infra.ipynb) -which provides different ways to run the model depending on your system and -[run_model_with_different_dataset.ipynb](./docs/notebooks/run_model_with_different_dataset.ipynb) -on how to use different datasets e.g. site or global data. - More information on the setup and installation, including for own machine, is available in the [documentation](https://pystemmusscope.readthedocs.io/). diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index b61deb3b..1c2cfa73 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,16 +1,81 @@ # Contributing Guide -If you want to contribute to `PyStemmusScope`, you will need to set up your development -environment the right way. The instructions below will guide you though the steps +We welcome any kind of contributions to our software, from simple +comment or question to a full fledged [pull +request](https://help.github.com/articles/about-pull-requests/). Please +read and follow our [Code of Conduct](./CODE_OF_CONDUCT.md). + +A contribution can be one of the following cases: + +1. you have a question; +2. you think you may have found a bug (including unexpected behavior); +3. you want to make some kind of change to the code base (e.g. to fix a + bug, to add a new feature, to update documentation). +4. you want to make a release + +The sections below outline the steps in each case. + +## You have a question + +1. use the search functionality + [here](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/issues) to see if + someone already filed the same issue; +2. if your issue search did not yield any relevant results, make a new issue; +3. apply the \"Question\" label; apply other labels when relevant. + +## You think you may have found a bug + +1. use the search functionality + [here](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/issues) to see + if someone already filed the same issue; +2. if your issue search did not yield any relevant results, make a new issue, + making sure to provide enough information to the rest of the community to + understand the cause and context of the problem. Depending on the issue, you + may want to include: - the [SHA + hashcode](https://help.github.com/articles/autolinked-references-and-urls/#commit-shas) + of the commit that is causing your problem; - some identifying information + (name and version number) for dependencies you\'re using; - information + about the operating system; +3. apply relevant labels to the newly created issue. + +## You want to make some kind of change to the code base + +1. (**important**) announce your plan to the rest of the community + *before you start working*. This announcement should be in the form + of a (new) issue; +2. (**important**) wait until some kind of consensus is reached about + your idea being a good idea; +3. if needed, fork the repository to your own Github profile and create your own + feature branch off of the latest main commit. While working on your feature + branch, make sure to stay up to date with the main branch by pulling in + changes, possibly from the \'upstream\' repository (follow the instructions + [here](https://help.github.com/articles/configuring-a-remote-for-a-fork/) + and [here](https://help.github.com/articles/syncing-a-fork/)); +4. If you are using [Visual Studio Code](https://code.visualstudio.com), some + extensions will be recommended and you are offered to run inside a + [DevContainer](https://containers.dev) in which the dependencies are already + installed; + +In case you feel like you\'ve made a valuable contribution, but you +don\'t know how to write or run tests for it, or how to generate the +documentation: don\'t let this discourage you from making the pull +request; we can help you! Just go ahead and submit the pull request, but +keep in mind that you might be asked to append additional commits to +your pull request. + +If you want to contribute to `PyStemmusScope`, we recommend installing +the package in editable mode. The instructions below will guide you though the steps required. -## Configure the python package for development and testing +### Dependencies -To contribute to the development of the python package, we recommend installing -the package in development mode. +Check the package `dependencies` and `optional dependencies` in the +`pyproject.toml` file in the root directory of the repository. The package +dependecies are those packages that are required to build the package as a +software. The optional dependencies are those packages that are required to run +the tests, build the documentation, and format the code. - -### Installation +### Installation in editable mode First, clone this repository: @@ -36,6 +101,13 @@ python3 -m pytest ### Building the documentation +To install the documentation dependencies (On Windows, use `python` instead of `python3`): + +```sh +cd STEMMUS_SCOPE_Processing +python3 -m install -e .[docs] +``` + To edit the documentation locally, do: ```sh @@ -43,16 +115,25 @@ mkdocs serve ``` Then open the local hyperlink displayed in the terminal, e.g.: + ``` INFO - [13:23:44] Serving on http://127.0.0.1:8000/ ``` ### Run formatting tools -You can use `prospector` to get information about errors, potential problems and convention violations. To run: +Formatting configs are listed in the `pyproject.toml` file. You can use `ruff` +to get information about errors, potential problems and convention violations. +To run: ```sh -prospector +ruff check . +``` + +It is possible to fix some of the errors automatically. To do so, run: + +```sh +ruff check --fix . ``` To format the import statements, you can use `isort` as: @@ -61,7 +142,69 @@ To format the import statements, you can use `isort` as: isort ``` -## Development of STEMMUS_SCOPE model +### BMI Developer instructions + +The Python BMI implemented in this package communicates with the Matlab code +through STDIN/STDOUT, or via a socket to the Docker container. +Over this interface, three commands can be sent to Matlab: + +1. `initialize "path_to_cfg_file.txt"` +2. `update` +3. `finalize` + +After the initialize and update steps, the Matlab process writes the state of +any BMI exposed variables to an hdf5-file in the directory of `OutputPath` as +defined in the configuration file. + +The Python BMI interfaces with this file to allow the variables to be read and set. + +#### Adding/changing exposed variables + +Step one of changing the exposed variables is to change the Matlab code and +generating a new MCR executable (and possibly Docker image). The exposed +variables are defined in +[`STEMMUS_SCOPE/src/STEMMUS_SCOPE_exe.m`](https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/src/STEMMUS_SCOPE_exe.m). +Under the `bmiVarNames` variable. Make sure that you add the model variable +here, as well as any info on the variable's grid. + +The available variable names (`MODEL_INPUT_VARNAMES`, `MODEL_OUTPUT_VARNAMES`), +their units (`VARNAME_UNITS`), datatypes (`VARNAME_DTYPE`) and grids +(`VARNAME_GRID`) are defined in constants at the top of the file +`PyStemmusScope/bmi/implementation.py`. These have to be updated to reflect the +changes in the state file. + +Lastly you have to update the `get_variable` and `set_variable` functions in +`PyStemmusScope/bmi/implementation.py`. Here you define how the python code can +access them. While writing the code you can inspect the state using +`model.state`, which allows you to view the full contents of the HDF5 file for +easier debugging. + +After implementing the BMI changes, a new [STEMMUS_SCOPE Docker +image](https://github.com/EcoExtreML/STEMMUS_SCOPE/pkgs/container/stemmus_scope) +should be released that is compatible with the new BMI implementation. A new +release usually includes a new tag. Then, you need to update the +`compatible_tags` variable of the class `StemmusScopeDocker` in +`PyStemmusScope/bmi/docker_process.py`. + +## Making a release + +This section describes how to make a release in 3 parts: + +1. preparation +1. making a release on GitHub + +### (1/2) Preparation + +1. Update the (don't forget to update links at bottom of page) +2. Verify that the information in `CITATION.cff` is correct, and that `.zenodo.json` contains equivalent data +3. Make sure the version has been updated. +4. Run the unit tests with `pytest -v` + +### (2/2) GitHub - -To contribute to the STEMMUS_SCOPE model, you need access to the model source code that is stored in the repository [STEMMUS_SCOPE](https://github.com/EcoExtreML/STEMMUS_SCOPE). +Don't forget to also make a [release on +GitHub](https://github.com/EcoExtreML/stemmus_scope_processing/releases/new). +This will trigger the github action `python-publish.yml` that publishes the +package on PyPI. If your repository uses the GitHub-Zenodo integration this will +also trigger Zenodo into making a snapshot of your repository and sticking a DOI +on it. diff --git a/docs/README.dev.md b/docs/README.dev.md deleted file mode 100644 index dd29486a..00000000 --- a/docs/README.dev.md +++ /dev/null @@ -1,214 +0,0 @@ -# `PyStemmusScope` developer documentation - -If you're looking for user documentation, go [here](../README.md). - -## Development install - -```shell -# Create a virtual environment, e.g. with -python3 -m venv env - -# activate virtual environment -source env/bin/activate - -# make sure to have a recent version of pip and setuptools -python3 -m pip install --upgrade pip setuptools - -# (from the project root directory) -# install PyStemmusScope as an editable package -python3 -m pip install --no-cache-dir --editable . -# install development dependencies -python3 -m pip install --no-cache-dir --editable .[dev] -``` - -Afterwards check that the install directory is present in the `PATH` environment variable. - -## Running the tests - -There are two ways to run tests. - -The first way requires an activated virtual environment with the development tools installed: - -```shell -pytest -v -``` - -The second is to use `tox`, which can be installed separately (e.g. with `pip install tox`), i.e. not necessarily inside the virtual environment you use for installing `PyStemmusScope`, but then builds the necessary virtual environments itself by simply running: - -```shell -tox -``` - -Testing with `tox` allows for keeping the testing environment separate from your development environment. -The development environment will typically accumulate (old) packages during development that interfere with testing; this problem is avoided by testing with `tox`. - -### Test coverage - -In addition to just running the tests to see if they pass, they can be used for coverage statistics, i.e. to determine how much of the package's code is actually executed during tests. -In an activated virtual environment with the development tools installed, inside the package directory, run: - -```shell -coverage run -``` - -This runs tests and stores the result in a `.coverage` file. -To see the results on the command line, run - -```shell -coverage report -``` - -`coverage` can also generate output in HTML and other formats; see `coverage help` for more information. - -## Running linters locally - -For linting we will use [prospector](https://pypi.org/project/prospector/) and to sort imports we will use -[isort](https://pycqa.github.io/isort/). Running the linters requires an activated virtual environment with the -development tools installed. - -```shell -# linter -prospector - -# recursively check import style for the PyStemmusScope module only -isort --recursive --check-only PyStemmusScope - -# recursively check import style for the PyStemmusScope module only and show -# any proposed changes as a diff -isort --recursive --check-only --diff PyStemmusScope - -# recursively fix import style for the PyStemmusScope module only -isort --recursive PyStemmusScope -``` - -To fix readability of your code style you can use [yapf](https://github.com/google/yapf). - -You can enable automatic linting with `prospector` and `isort` on commit by enabling the git hook from `.githooks/pre-commit`, like so: - -```shell -git config --local core.hooksPath .githooks -``` - -## Generating the API docs - -```shell -cd docs -make html -``` - -The documentation will be in `docs/_build/html` - -If you do not have `make` use - -```shell -sphinx-build -b html docs docs/_build/html -``` - -To find undocumented Python objects run - -```shell -cd docs -make coverage -cat _build/coverage/python.txt -``` - -To [test snippets](https://www.sphinx-doc.org/en/master/usage/extensions/doctest.html) in documentation run - -```shell -cd docs -make doctest -``` - -## Versioning - -Bumping the version across all files is done with [bumpversion](https://github.com/c4urself/bump2version), e.g. - -```shell -bumpversion major -bumpversion minor -bumpversion patch -``` - -## Making a release - -This section describes how to make a release in 3 parts: - -1. preparation -1. making a release on PyPI -1. making a release on GitHub - -### (1/3) Preparation - -1. Update the (don't forget to update links at bottom of page) -2. Verify that the information in `CITATION.cff` is correct, and that `.zenodo.json` contains equivalent data -3. Make sure the [version has been updated](#versioning). -4. Run the unit tests with `pytest -v` - -### (2/3) PyPI - -In a new terminal, without an activated virtual environment or an env directory: - -```shell -# prepare a new directory -cd $(mktemp -d pystemmusscope.XXXXXX) - -# fresh git clone ensures the release has the state of origin/main branch -git clone https://github.com/EcoExtreML/stemmus_scope_processing . - -# prepare a clean virtual environment and activate it -python3 -m venv env -source env/bin/activate - -# make sure to have a recent version of pip and setuptools -python3 -m pip install --upgrade pip setuptools - -# install runtime dependencies and publishing dependencies -python3 -m pip install --no-cache-dir . -python3 -m pip install --no-cache-dir .[publishing] - -# clean up any previously generated artefacts -rm -rf PyStemmusScope.egg-info -rm -rf dist - -# create the source distribution and the wheel -python3 setup.py sdist bdist_wheel - -# upload to test pypi instance (requires credentials) -twine upload --repository-url https://test.pypi.org/legacy/ dist/* -``` - -Visit -[https://test.pypi.org/project/PyStemmusScope](https://test.pypi.org/project/PyStemmusScope) -and verify that your package was uploaded successfully. Keep the terminal open, we'll need it later. - -In a new terminal, without an activated virtual environment or an env directory: - -```shell -cd $(mktemp -d PyStemmusScope-test.XXXXXX) - -# prepare a clean virtual environment and activate it -python3 -m venv env -source env/bin/activate - -# make sure to have a recent version of pip and setuptools -pip install --upgrade pip setuptools - -# install from test pypi instance: -python3 -m pip -v install --no-cache-dir \ ---index-url https://test.pypi.org/simple/ \ ---extra-index-url https://pypi.org/simple PyStemmusScope -``` - -Check that the package works as it should when installed from pypitest. - -Then upload to pypi.org with: - -```shell -# Back to the first terminal, -# FINAL STEP: upload to PyPI (requires credentials) -twine upload dist/* -``` - -### (3/3) GitHub - -Don't forget to also make a [release on GitHub](https://github.com/EcoExtreML/stemmus_scope_processing/releases/new). If your repository uses the GitHub-Zenodo integration this will also trigger Zenodo into making a snapshot of your repository and sticking a DOI on it. diff --git a/docs/bmi.md b/docs/bmi.md index 3ce9dd0d..aabbbf2a 100644 --- a/docs/bmi.md +++ b/docs/bmi.md @@ -1,86 +1,85 @@ # Basic Model Interface -The [Basic Model Interface](https://csdms.colorado.edu/wiki/BMI) is a standard way of communicating with models. -PyStemmusScope implements the Basic Model Interface for STEMMUS_SCOPE. -There are multiple ways to run the STEMMUS_SCOPE Basic Model Interface. -For the model, we generated Matlab Compiler Runtime executable file (only available for x86 Linux). -This requires installation of MCR. -The other option is to use the Dockerized version of the executable, available on ghcr.io/ecoextreml/stemmus_scope. - -For more information on each method, see the sections below. +The [Basic Model Interface](https://csdms.colorado.edu/wiki/BMI) is a standard +way of communicating with models. PyStemmusScope implements the Basic Model +Interface for STEMMUS_SCOPE. There are multiple ways to run the STEMMUS_SCOPE +Basic Model Interface. For the model, we generated Matlab Compiler Runtime +executable file (only available for x86 Linux). This requires installation of +MCR. The other option is to use the Dockerized version of the executable, +available on +[https://ghcr.io/ecoextreml/stemmus_scope](https://ghcr.io/ecoextreml/stemmus_scope). For more +information on each method, see the sections below. ## Installation and setup ### Dockerized executable -STEMMUS_SCOPE has a Docker image available. This allows you to run the executable file without having to install MCR. -The Docker image is available at https://ghcr.io/ecoextreml/stemmus_scope. The Docker image is created using the docker file [here](https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/Dockerfile). -To use the Docker image, use the `DockerImage` setting in the configuration file: +STEMMUS_SCOPE has a Docker image available. This allows you to run the +executable file without having to install MCR. The Docker image is available at +[https://ghcr.io/ecoextreml/stemmus_scope](https://ghcr.io/ecoextreml/stemmus_scope). +The Docker image is created using the docker file +[here](https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/Dockerfile). To use +the Docker image, use the `DockerImage` setting in the configuration file: + ```sh WorkDir=/home/username/tmp/stemmus_scope ... DockerImage=ghcr.io/ecoextreml/stemmus_scope:1.6.2 ``` -It is best to add the version tag here too (`:1.6.2`), this way the BMI will warn you if the version might be incompatible. - -Note that the `docker` package for python is required here. Install this with `pip install PyStemmusScope[docker]`. -Additionally, [Docker](https://docs.docker.com/get-docker/) itself has to be installed. +It is best to add the version tag here too (`:1.6.2`), this way the BMI will +warn you if the version might be incompatible. Note that the `docker` package +for python is required. Install this with `pip install PyStemmusScope[docker]`. +Additionally, [Docker](https://docs.docker.com/get-docker/) itself has to be +installed. ### Local executable file -The executable file can be downloaded from the STEMMUS_SCOPE repository. More specifically [here](https://github.com/EcoExtreML/STEMMUS_SCOPE/tree/main/run_model_on_snellius/exe). -To be able to run this executable, you need a Linux x86 system, along with Matlab Compiler Runtime R2023a. MCR is available [here](https://nl.mathworks.com/products/compiler/matlab-runtime.html). +You can run the model using the executable file. It can be downloaded from the +STEMMUS_SCOPE repository, +[here](https://github.com/EcoExtreML/STEMMUS_SCOPE/tree/main/run_model_on_snellius/exe). +To be able to run this executable, you need a Linux x86 system, along with +Matlab Compiler Runtime R2023a. MCR is available +[here](https://nl.mathworks.com/products/compiler/matlab-runtime.html). To use +the local executable file, add the path to the executable file to the config +file. E.g.: -To use the local executable file, add the path to the executable file to the config file. E.g.: ``` WorkDir=/home/username/tmp/stemmus_scope ... ExeFilePath=/path/to/executable/STEMMUS_SCOPE ``` -Alternatively, if the environmental variable `STEMMUS_SCOPE` is configured, the BMI will use this if the ExeFilePath or DockerImage are not set in the configuration file. +Alternatively, if the environmental variable `STEMMUS_SCOPE` is configured, the +BMI will use this if the ExeFilePath or DockerImage are not set in the +configuration file. ## Using the BMI -A [notebook demonstration the use of the Basic Model Interface](notebooks/BMI_demo.ipynb) is available. -For more information on using BMI, see the [CSDMS website](https://csdms.colorado.edu/wiki/BMI). - -If you need access to other model variables that are not yet available in the BMI, please raise an issue on the [STEMMUS_SCOPE repository](https://github.com/EcoExtreML/STEMMUS_SCOPE/issues), or leave a comment if an issue is open already. - -## grpc4bmi - -A [Docker image is available](https://ghcr.io/ecoextreml/stemmus_scope-grpc4bmi) in which the model as well as the Python BMI have been wrapped in a container. The Docker image is created using the Docker file [here](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/Dockerfile) and allows communication with a STEMMUS_SCOPE BMI through [grpc4bmi](https://grpc4bmi.readthedocs.io/en/latest/). - -Doing so avoids the needs to install PyStemmusScope yourself, only Docker/apptainer and a python environment with grpc4bmi are required. Please note you should not specify `DockerImage` or `ExeFilePath` in the config file if you are using the grpc4bmi interface. - -A demonstration is available [here](notebooks/grpc4bmi_demo.ipynb) +A [notebook demonstration the use of the Basic Model +Interface](notebooks/BMI_demo.ipynb) is available. For more information on using +BMI, see the [CSDMS website](https://csdms.colorado.edu/wiki/BMI). + +If you need access to other model variables that are not yet available in the +BMI, please raise an issue on the [STEMMUS_SCOPE +repository](https://github.com/EcoExtreML/STEMMUS_SCOPE/issues), or leave a +comment if an issue is open already. + +## Using grpc4bmi + +A [Docker image is available](https://ghcr.io/ecoextreml/stemmus_scope-grpc4bmi) +in which the model as well as the Python BMI have been wrapped in a container. +The Docker image is created using the Docker file +[here](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/Dockerfile) +and allows communication with a STEMMUS_SCOPE BMI through +[grpc4bmi](https://grpc4bmi.readthedocs.io/en/latest/). Doing so avoids the +needs to install PyStemmusScope yourself, only Docker/apptainer and a python +environment with grpc4bmi are required. Please note you should not specify +`DockerImage` or `ExeFilePath` in the config file if you are using the grpc4bmi +interface. Only set the `STEMMUS_SCOPE` environmental variable with a path to +the executable file. A demonstration is available +[here](notebooks/grpc4bmi_demo.ipynb). ## Developer instructions -The Python BMI implemented in this package communicates with the Matlab code through STDIN/STDOUT, or via a socket to the Docker container. -Over this interface, three commands can be sent to Matlab: - -1. `initialize "path_to_cfg_file.txt"` -2. `update` -3. `finalize` - -After the initialize and update steps, the Matlab process writes the state of any BMI exposed variables to an hdf5-file in the directory of `OutputPath` as defined in the configuration file. - -The Python BMI interfaces with this file to allow the variables to be read and set. - -### Adding/changing exposed variables - -Step one of changing the exposed variables is to change the Matlab code and generating a new MCR executable (and possibly Docker image). -The exposed variables are defined in [`STEMMUS_SCOPE/src/STEMMUS_SCOPE_exe.m`](https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/src/STEMMUS_SCOPE_exe.m). -Under the `bmiVarNames` variable. -Make sure that you add the model variable here, as well as any info on the variable's grid. - -The available variable names (`MODEL_INPUT_VARNAMES`, `MODEL_OUTPUT_VARNAMES`), their units (`VARNAME_UNITS`), datatypes (`VARNAME_DTYPE`) and grids (`VARNAME_GRID`) are defined in constants at the top of the file `PyStemmusScope/bmi/implementation.py`. -These have to be updated to reflect the changes in the state file. - -Lastly you have to update the `get_variable` and `set_variable` functions in `PyStemmusScope/bmi/implementation.py`. -Here you define how the python code can access them. -While writing the code you can inspect the state using `model.state`, which allows you to view the full contents of the HDF5 file for easier debugging. - -After implementing the BMI changes, a new [STEMMUS_SCOPE Docker image](https://github.com/EcoExtreML/STEMMUS_SCOPE/pkgs/container/stemmus_scope) should be released that is compatible with the new BMI implementation. A new release usually includes a new tag. Then, you need to update the `compatible_tags` variable of the class `StemmusScopeDocker` in `PyStemmusScope/bmi/docker_process.py`. +Follow the instructions in the `Contributing Guide`. diff --git a/docs/downloading_global_data.md b/docs/downloading_global_data.md new file mode 100644 index 00000000..d211a3e4 --- /dev/null +++ b/docs/downloading_global_data.md @@ -0,0 +1,62 @@ +# Global data for STEMMUS_SCOPE + +This document outlines which, where and how we download the "global" input data for the +model. + +## Download and prepare data + +Tha python package [`zampy`](https://zampy.readthedocs.io/) can be used to +download and prepare the data. + +## Data of Climate Data store (CDS) + +Many of the forcing-related data is available in the era5 reanalysis data. + +**era5 variables**: + +- mean_total_precipitation_rate +- surface_thermal_radiation_downwards +- surface_solar_radiation_downwards +- surface_pressure +- 10m_u_component_of_wind +- 10m_v_component_of_wind + +**era5-land variables**: + +- 2m_temperature +- 2m_dewpoint_temperature + +**era5-land soil initial conditions**: + +For running STEMMUS-SCOPE, global data is also required for the soil initial conditions. These are retrieved from ERA5-land. + +## CO2 data from Atmosphere Data Store (ADS) + +CO2 data is available in the CAMS dataset. A simple check for the parsing of the +data is in `global_data/data_analysis_notebooks/parse_CO2_data.ipynb`. + +## Canopy height data from ETH + +The canopy height data is described in: +[https://langnico.github.io/globalcanopyheight/](https://langnico.github.io/globalcanopyheight/) +and available +[here](https://share.phys.ethz.ch/~pf/nlangdata/ETH_GlobalCanopyHeight_10m_2020_version1/3deg_cogs/). +A simple example for the parsing of the data is in +`global_data/data_analysis_notebooks/parse_canopy_height.ipynb`. + +## DEM data from Copernicus + +DEM data is provided by Copernicus, see +[here](https://dataspace.copernicus.eu/explore-data/data-collections/copernicus-contributing-missions/collections-description/COP-DEM). +A simple example for the parsing of the data is in +`global_data/data_analysis_notebooks/parse_dem.ipynb`. + +## LAI from Climate Data Store (CDS) + +LAI data was retrieved from the CDS. However, there are some downloading issues +with the `satellite-lai-fapar` dataset. A simple example for parsing the LAI +data is in `global_data/data_analysis_notebooks/parse_LAI.py`. + +## Land cover from Climate Data Store (CDS) + +Land cover data is available at [https://cds.climate.copernicus.eu/datasets/satellite-land-cover](https://cds.climate.copernicus.eu/datasets/satellite-land-cover). diff --git a/docs/ecoextreml_dt.png b/docs/ecoextreml_dt.png new file mode 100644 index 00000000..03546a57 Binary files /dev/null and b/docs/ecoextreml_dt.png differ diff --git a/docs/getting_started.md b/docs/getting_started.md new file mode 100644 index 00000000..eda097e1 --- /dev/null +++ b/docs/getting_started.md @@ -0,0 +1,39 @@ +# + +This is the documentation for the python package `PyStemmusScope`, which allows +for preparing data and running the STEMMUS-SCOPE model. The model source code +[STEMMUS_SCOPE repository](https://github.com/EcoExtreML/STEMMUS_SCOPE). + +## Requirements + +To run the model, check the +[requirements](https://ecoextreml.github.io/STEMMUS_SCOPE/getting_started/). + +### Global data + +See [this documentation](./downloading_global_data.md) on which, where and how +we download the "global" input data for the model. + +## Configuration file + +The configuration file is a text file that sets the paths required by the model. +Check [**required** information and +templates](https://ecoextreml.github.io/STEMMUS_SCOPE/getting_started/#configuration-file). +In addition to required information, there are optional parameters that can be +set in the configuration file: + +- `soil_layers_thickness`: a path to a csv file containing soil layers thickness + information, see + [exmaple](https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/example_data/input_soilLayThick.csv). +- `ExeFilePath`: a path to the STEMMUS-SCOPE executable file, if BMI interface + is used. +- `DockerImage`: a path to the Docker image, if BMI interface and docker are + used. +- `SleepDuration`: a time in seconds to wait before checking if the model has + finished running in BMI. Default is 10 seconds. + +## Running the model + +If you want to run the model using `PyStemmusScope`, follow the instructions in +the `installation` and `Run the model` documentation. If you want to add changes +to the package `PyStemmusScope`, follow `Contributing guide` documnetation. diff --git a/docs/index.md b/docs/index.md index abff24a1..22f2a112 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,29 +1,12 @@ -Logo +# -# Getting started +The python package `PyStemmusScope` allows for preparing data and running the +STEMMUS-SCOPE model, where STEMMUS simulates soil water and heat, and the model +SCOPE the vegetation photosynthesis and land surface energy balance. In +addition, it supports the use of the Basic Model Interface (BMI) to couple the +model with other models. -This is the documentation for the python package `PyStemmusScope`, which allows for for -running the STEMMUS-SCOPE model. - - -The model source code, executable file and utility files are available in the - -[STEMMUS_SCOPE repository](https://github.com/EcoExtreML/STEMMUS_SCOPE). - -The input datasets are available on Snellius and CRIB. First, make sure you have -right access to the repository and data. - -## Running the model - -1. Using executable file: As a user, you don't need to have a MATLAB license to -run the STEMMUS-SCOPE model. If `PyStemmusScope` and `MATLAB Runtime` are -installed on a Unix-like system (e.g. your own machine, Snellius or WSL), you -can run STEMMUS_SCOPE using the executable file. -2. Using Matlab: If `PyStemmusScope` and `Matlab` are installed, you can run -STEMMUS_SCOPE from the source code, for example on Snellius or CRIB. -3. Using Octave: If `PyStemmusScope` and latest `Octave` including required -packages are installed, you can run STEMMUS_SCOPE from its source code, for -example on CRIB or your own machine. - -See section [Installation Instructions](installation_instructions.md) for required packages. +![Towards a digital twin of soil-plant system](ecoextreml_dt.png) +The python package `PyStemmusScope` is developed in [EcoExtreML +project](https://research-software-directory.org/projects/ecoextreml). diff --git a/docs/installation_instructions.md b/docs/installation_instructions.md index 10a07ed0..37b209e8 100644 --- a/docs/installation_instructions.md +++ b/docs/installation_instructions.md @@ -1,32 +1,17 @@ # Installation instructions -The installation instructions depend on which computers you want to run the model, be it -Snellius, CRIB, or your own local machine. -## On Snellius +This is the installation instruction for the python package `PyStemmusScope`, +which allows for preparing data and running the STEMMUS-SCOPE model. The model +source code [STEMMUS_SCOPE +repository](https://github.com/EcoExtreML/STEMMUS_SCOPE). For model-specific +instructions, check the `Getting started` page. -[Snellius](https://servicedesk.surfsara.nl/wiki/display/WIKI/Snellius) is the -Dutch National supercomputer hosted at SURF. MATLAB and MATLAB Runtime are -installed on Snellius, see the script -[`run_jupyter_lab_snellius.sh`](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/run_jupyter_lab_snellius.sh) -on how to load the module. Also, use the same script to create a jupyter lab -server for running notebooks interactively. The script activates the conda -environment `pystemmusscope`. Make sure that you create the `pystemmusscope` -conda environment before submitting the the bash script. See -[Create pystemmusscope conda environment](#create-pystemmusscope-conda-environment). - -## On CRIB - -[CRIB](https://crib.utwente.nl/) is the ITC Geospatial Computing Platform. You -can run the model using `Matlab` or `Octave`. Currently, running the -exceutable file on CRIB is not supported because MATLAB Runtime can not be -installed there. See [Install PyStemmusScope](#install-pystemmusscope). - -## On your own machine - -Choose how do you want to run the model, see [Run the model](notebooks/run_model_on_different_infra.ipynb). +Note that the latest version of `PyStemmusScope` is compatible with latest +version of `STEMMUS-SCOPE` model. ## Install PyStemmusScope +To install the package, you need to have Python ">=3.9, <3.12" installed. Run the commands below in a terminal (On Windows, use `python` instead of `python3`): @@ -42,6 +27,12 @@ Open a jupyter notebook and run the code below in a cell: !pip install pystemmusscope ``` +On CRIB, you can use the following command to install the package: + +```sh +python3 -m pip install --user pystemmusscope +``` + ## Install jupyterlab Jupyterlab is needed to run notebooks. Run the commands below in a terminal: @@ -61,29 +52,7 @@ JupyterLab will open automatically in your browser. Now, you can run the notebook [run_model_on_different_infra.ipynb](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/docs/notebooks/run_model_on_different_infra.ipynb). -## Install MATLAB Runtime - -To run the STEMMUS_SCOPE, you need MATLAB Runtime version `2021a` and a Unix-like system. - -In a terminal: - -```sh -# Download MATLAB Runtime for Linux -wget https://ssd.mathworks.com/supportfiles/downloads/R2021a/Release/6/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021a_Update_6_glnxa64.zip - -# Unzip the file -unzip MATLAB_Runtime_R2021a_Update_6_glnxa64.zip -d MATLAB_Runtime - -# Install it -cd MATLAB_Runtime -sudo -H ./install -mode silent -agreeToLicense yes -``` - -For more information on how to download and install MATLAB Runtime, see the links below: - - [download](https://nl.mathworks.com/products/compiler/matlab-runtime.html) - - [installation](https://nl.mathworks.com/help/compiler/install-the-matlab-runtime.html) - -## Install WSL +## Install WSL [optional] As the STEMMUS-SCOPE executable only supports Unix-like systems, Windows users cannot run STEMMUS-SCOPE natively. However, users of Windows 10 and newer can @@ -114,9 +83,10 @@ distro. However, WSL does not have write permission. Therefore, output data will be stored within WSL. Make sure that `WorkDir` in the model config file is set correctly. -## Create pystemmusscope conda environment +## Create pystemmusscope conda environment [optional] -If a conda environment is needed, run the commands below in a terminal: +If a conda environment is needed, for example, on Snellius, run the commands +below in a terminal: ```sh # Download and install Mamba on linux diff --git a/docs/notebooks/bmi_MODFLOW_coupling.ipynb b/docs/notebooks/bmi_MODFLOW_coupling.ipynb index 17718a88..e17fa949 100644 --- a/docs/notebooks/bmi_MODFLOW_coupling.ipynb +++ b/docs/notebooks/bmi_MODFLOW_coupling.ipynb @@ -6,7 +6,7 @@ "source": [ "# STEMMUS_SCOPE & MODFLOW 6 BMI coupling\n", "\n", - "This Python notebook shows a simple example of BMI coupling between the STEMMUS_SCOPE model and the MODFLOW 6 model. The example is prepared for one of the PLUMBER2 dataset (https://researchdata.edu.au/plumber2-forcing-evaluation-surface-models/1656048): the LooBos site, the Netherlands (NL-Loo). To run this notebook, the following are required:\n", + "This Python notebook shows a simple example of BMI coupling between the STEMMUS_SCOPE model and the MODFLOW 6 model, based on research carried by \"M. Daoud\", as a part of [EcoExtreML](https://research-software-directory.org/projects/ecoextreml) project, for more info see [this reference](https://research.utwente.nl/en/publications/investigating-the-role-of-groundwater-in-mitigating-vegetation-wa). The example is prepared for one of the PLUMBER2 dataset (https://researchdata.edu.au/plumber2-forcing-evaluation-surface-models/1656048): the LooBos site, the Netherlands (NL-Loo). To run this notebook, the following are required:\n", "- Install the related packages:\n", " - pyStemmusScope package (https://pystemmusscope.readthedocs.io/en/latest/).\n", " - modflowapi Python package (https://github.com/MODFLOW-USGS/modflowapi?tab=readme-ov-file#installation).\n", @@ -124,11 +124,11 @@ } ], "source": [ - "# Add the matlab runtime compiler locations: \n", + "# Add the matlab runtime compiler locations:\n", "matlab_version = 'R2023a'\n", - "matlab_path = '/opt/matlab/MATLAB_Runtime/' + matlab_version \n", - " \n", - "os.environ['LD_LIBRARY_PATH'] = ( \n", + "matlab_path = '/opt/matlab/MATLAB_Runtime/' + matlab_version\n", + "\n", + "os.environ['LD_LIBRARY_PATH'] = (\n", " f\"{matlab_path}/runtime/glnxa64:\"\n", " f\"{matlab_path}/bin/glnxa64:\"\n", " f\"{matlab_path}/sys/os/glnxa64:\"\n", @@ -262,7 +262,7 @@ ], "source": [ "sim = ApiSimulation.load(modflow)\n", - "gwf_name = sim.model_names[0] # groundwater flow model \n", + "gwf_name = sim.model_names[0] # groundwater flow model\n", "gwe_name = sim.model_names[1] # groundwater energy (heat) model\n", "gwf_name, gwe_name" ] @@ -399,7 +399,7 @@ "source": [ "stemmus_scope.set_value(\"groundwater_elevation_top_aquifer\", top_elev)\n", "stemmus_scope.set_value(\"groundwater_head_bottom_layer\", gwhead_init)\n", - "stemmus_scope.set_value(\"groundwater_temperature\", gwtemp_init) " + "stemmus_scope.set_value(\"groundwater_temperature\", gwtemp_init)" ] }, { @@ -6650,9 +6650,9 @@ ], "source": [ "while modflow.get_current_time() < modflow.get_end_time():\n", - " while stemmus_scope.get_current_time() < stemmus_scope.get_end_time(): \n", - " \n", - " # Get STEMMUS_SCOPE variables (soil moisture, soil temperature, groundwater recharge) \n", + " while stemmus_scope.get_current_time() < stemmus_scope.get_end_time():\n", + "\n", + " # Get STEMMUS_SCOPE variables (soil moisture, soil temperature, groundwater recharge)\n", " stemmus_scope.get_value(\"soil_moisture\", soil_moisture[i])\n", " stemmus_scope.get_value(\"soil_temperature\", soil_temperature[i])\n", " stemmus_scope.get_value(\"groundwater_recharge\", gw_recharge)\n", @@ -6669,19 +6669,19 @@ " # Set STEMMUS_SCOPE variables (groundwater head, groundwater temperature)\n", " stemmus_scope.set_value(\"groundwater_head_bottom_layer\", gwhead)\n", " stemmus_scope.set_value(\"groundwater_temperature\", gwtemp)\n", - " \n", + "\n", " # Set MODFLOW 6 variables (groundwater recharge)\n", - " kstp, kper = sim.kstp, sim.kper \n", + " kstp, kper = sim.kstp, sim.kper\n", " if kstp == nstp - 1:\n", - " # groundwater recharge per stress period = sum of groundwater recharge per time step \n", + " # groundwater recharge per stress period = sum of groundwater recharge per time step\n", " gw_recharge_nper = np.array([-1 * np.sum(gw_recharge_nstp) / 100]) # multiply by -1 due to different signs of both models\n", " modflow.set_value(rch_tag, gw_recharge_nper)\n", - " gw_recharge_nstp = np.array([]) \n", - " \n", - " # Track the simulation time of MODFLOW 6 \n", + " gw_recharge_nstp = np.array([])\n", + "\n", + " # Track the simulation time of MODFLOW 6\n", " print('Updating MODFLOW Stress period ' + str(kper + 1) + ', time step ' + str(kstp + 1))\n", " # print(kper + 1, kstp + 1, gwhead, gwtemp)\n", - " \n", + "\n", " # Update models\n", " stemmus_scope.update()\n", " modflow.update()" diff --git a/docs/notebooks/run_model_on_different_infra.ipynb b/docs/notebooks/run_model_on_different_infra.ipynb index cf7c5402..0599b4f3 100644 --- a/docs/notebooks/run_model_on_different_infra.ipynb +++ b/docs/notebooks/run_model_on_different_infra.ipynb @@ -5,10 +5,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Running the STEMMUS_SCOPE model on a system\n", + "This notebook shows steps to run the STEMMUS_SCOPE model, including preprocessing and postprocessing. For postprocesisng, the file `required_netcdf_variables.csv` can be found at https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/utils/csv_to_nc/required_netcdf_variables.csv. \n", "\n", - "This notebook shows steps to run the STEMMUS_SCOPE model, including preprocessing and postprocessing. STEMMUS_SCOPE files are located in a repository https://github.com/EcoExtreML/STEMMUS_SCOPE. Then specify the path to a config file e.g.`config_file_template.txt`, executable file `STEMMUS_SCOPE` or source code `STEMMUS_SCOPE/src` in the cells below. There are already config files for users on Snellius and CRIB, see `config_file_snellius.txt`, `config_file_crib.txt`. Depending on your system (OS, Matlab availability, etc.), there are a few ways you can run the model. Choose how do you want to run the model:\n", + "First, check the `Getting started` page on documentation of [STEMMUS_SCOPE](https://ecoextreml.github.io/STEMMUS_SCOPE/getting_started/) and [pystemmusscope](https://pystemmusscope.readthedocs.io/en/latest/). \n", "\n", + "Depending on your system (OS, Matlab availability, etc.), there are a few ways you can run the model. Choose how do you want to run the model:\n", "\n", "[**Executable file**](#1-using-executable-file)\n", "\n", @@ -112,7 +113,7 @@ " EndTime=\"2001-01-02T01:30\",\n", ")\n", "\n", - "# new config file genertaed to run the model \n", + "# new config file genertaed to run the model\n", "print(f\"New config file {config_path}\")\n", "\n", "# see input and output paths generated by the model\n", @@ -217,7 +218,7 @@ " EndTime=\"2001-01-02T01:30\",\n", ")\n", "\n", - "# new config file genertaed to run the model \n", + "# new config file genertaed to run the model\n", "print(f\"New config file {config_path}\")\n", "\n", "# see input and output paths generated by the model\n", @@ -340,7 +341,7 @@ " EndTime=\"2001-01-02T01:30\",\n", ")\n", "\n", - "# new config file genertaed to run the model \n", + "# new config file genertaed to run the model\n", "print(f\"New config file {config_path}\")\n", "\n", "# see input and output paths generated by the model\n", diff --git a/docs/notebooks/run_model_with_different_dataset.ipynb b/docs/notebooks/run_model_with_different_dataset.ipynb index b5202e5f..b49bd1f1 100644 --- a/docs/notebooks/run_model_with_different_dataset.ipynb +++ b/docs/notebooks/run_model_with_different_dataset.ipynb @@ -5,12 +5,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Running the STEMMUS_SCOPE model with different dataset\n", + "This notebook shows how to run the STEMMUS_SCOPE model uisng the `Executable` file, including preprocessing and postprocessing. See the [Different ways to run the model](./run_model_on_different_infra.ipynb) for different ways e.g. Executable file, Matlab, Octave.\n", "\n", - "This notebook shows how to run the STEMMUS_SCOPE model uisng the `Executable` file, including preprocessing and postprocessing. For postprocesisng, the file `required_netcdf_variables.csv` can be found at https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/utils/csv_to_nc/required_netcdf_variables.csv.\n", + "Also, check the `Getting started` page on documentation of [STEMMUS_SCOPE](https://ecoextreml.github.io/STEMMUS_SCOPE/getting_started/) and [pystemmusscope](https://pystemmusscope.readthedocs.io/en/latest/). \n", "\n", - "See the [Running the STEMMUS_SCOPE model on a system](./run_model_on_different_infra.ipynb) for different ways e.g. Executable file, Matlab, Octave.\n", - "The `STEMMUS_SCOPE` executable file is located in a repository https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/run_model_on_snellius/exe/STEMMUS_SCOPE.\n", + "For postprocesisng, the file `required_netcdf_variables.csv` can be found at https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/utils/csv_to_nc/required_netcdf_variables.csv.\n", "\n", "## Which datasets are available?\n", "The config file shows how to set the paths to different data needed by the model. Check the data source on Snelius and CRIB and modify the config file accordingly:\n", @@ -136,7 +135,7 @@ " EndTime=\"2001-01-02T01:30\",\n", ")\n", "\n", - "# new config file genertaed to run the model \n", + "# new config file genertaed to run the model\n", "print(f\"New config file {config_path}\")\n", "\n", "# see input and output paths generated by the model\n", @@ -296,7 +295,7 @@ " EndTime=\"2014-01-15T00:00\",\n", ")\n", "\n", - "# new config file genertaed to run the model \n", + "# new config file genertaed to run the model\n", "print(f\"New config file {config_path}\")\n", "\n", "# see input and output paths generated by the model\n", diff --git a/docs/notebooks/run_model_with_different_dataset_on_CRIB.ipynb b/docs/notebooks/run_model_with_different_dataset_on_CRIB.ipynb index a668edb9..0fe818b2 100644 --- a/docs/notebooks/run_model_with_different_dataset_on_CRIB.ipynb +++ b/docs/notebooks/run_model_with_different_dataset_on_CRIB.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Running the STEMMUS_SCOPE model with different dataset on CRIB using MATLAB\n", + "This notebook shows how to run the STEMMUS_SCOPE model uisng source code, including preprocessing and postprocessing. See the [Different ways to run the model](./run_model_on_different_infra.ipynb) for different ways e.g. Executable file, Matlab, Octave.\n", "\n", - "This notebook shows how to run the STEMMUS_SCOPE model uisng source code, including preprocessing and postprocessing. For postprocesisng, the file required_netcdf_variables.csv can be found at https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/utils/csv_to_nc/required_netcdf_variables.csv.\n", + "Also, check the `Getting started` page on documentation of [STEMMUS_SCOPE](https://ecoextreml.github.io/STEMMUS_SCOPE/getting_started/) and [pystemmusscope](https://pystemmusscope.readthedocs.io/en/latest/). \n", "\n", - "See the [Running the STEMMUS_SCOPE model on a system](./run_model_on_different_infra.ipynb) for different ways e.g. Executable file, Matlab, Octave.\n", + "For postprocesisng, the file `required_netcdf_variables.csv` can be found at https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/utils/csv_to_nc/required_netcdf_variables.csv.\n", "\n", "## Which datasets are available?\n", "The config file shows how to set the paths to different data needed by the model. Check the data source on Snelius and CRIB and modify the config file accordingly:\n", @@ -100,7 +100,7 @@ " EndTime=\"2001-01-02T01:30\",\n", ")\n", "\n", - "# new config file genertaed to run the model \n", + "# new config file genertaed to run the model\n", "print(f\"New config file {config_path}\")\n", "\n", "# see input and output paths generated by the model\n", @@ -227,7 +227,7 @@ " EndTime=\"2014-01-15T00:00\",\n", ")\n", "\n", - "# new config file genertaed to run the model \n", + "# new config file genertaed to run the model\n", "print(f\"New config file {config_path}\")\n", "\n", "# see input and output paths generated by the model\n", @@ -262,20 +262,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "b'MATLAB is selecting SOFTWARE OPENGL rendering.\\nOpening log file: /home/alidoost/java.log.37351\\n\\n < M A T L A B (R) >\\n Copyright 1984-2021 The MathWorks, Inc.\\n R2021a Update 3 (9.10.0.1684407) 64-bit (glnxa64)\\n May 27, 2021\\n\\n \\nTo get started, type doc.\\nFor product information, visit www.mathworks.com.\\n \\nReading config from /gpfs/home2/alidoost/test_matlab/input/ZA-Kru_2022-10-27-1651/ZA-Kru_2022-10-27-1651_config.txt\\nThe calculations start now\\nThe calculations end now\\n'" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# run the model\n", "result = model.run()\n", diff --git a/docs/project_setup.md b/docs/project_setup.md index f05cc9e3..eb599238 100644 --- a/docs/project_setup.md +++ b/docs/project_setup.md @@ -7,28 +7,14 @@ text when the development of the software package takes off. For a quick reference on software development, we refer to [the software guide checklist](https://guide.esciencecenter.nl/#/best_practices/checklist). -## Python versions - -This repository is set up with Python versions: - -- 3.9 -- 3.10 -- 3.11 - -Add or remove Python versions based on project requirements. See [the -guide](https://guide.esciencecenter.nl/#/best_practices/language_guides/python) for more information about Python -versions. - ## Package management and dependencies -You can use either pip or conda for installing dependencies and package management. This repository does not force you -to use one or the other, as project requirements differ. For advice on what to use, please check [the relevant section -of the +You can use either pip or conda for installing dependencies (see +`pyproject.toml`) and package management. This repository does not force you to +use one or the other, as project requirements differ. For advice on what to use, +please check [the relevant section of the guide](https://guide.esciencecenter.nl/#/best_practices/language_guides/python?id=dependencies-and-package-management). -- Runtime dependencies should be added to `setup.cfg` in the `install_requires` list under `[options]`. -- Development dependencies should be added to `setup.cfg` in one of the lists under `[options.extras_require]`. - ## Packaging/One command install You can distribute your code using PyPI. @@ -43,33 +29,18 @@ help you decide which tool to use for packaging. - The testing framework used is [PyTest](https://pytest.org) - [PyTest introduction](https://pythontest.com/pytest-book/) - PyTest is listed as a development dependency - - This is configured in `setup.cfg` + - This is configured in `pyproject.toml` - The project uses GitHub action workflows to automatically run tests on GitHub infrastructure against multiple Python versions - Workflows can be found in [`.github/workflows`](../.github/workflows/) - [Relevant section in the guide](https://guide.esciencecenter.nl/#/best_practices/language_guides/python?id=testing) ## Documentation -- Documentation should be put in the [`docs/`](../docs) directory. The contents have been generated using `sphinx-quickstart` (Sphinx version 1.6.5). -- We recommend writing the documentation using Restructured Text (reST) and Google style docstrings. - - [Restructured Text (reST) and Sphinx CheatSheet](https://thomas-cokelaer.info/tutorials/sphinx/rest_syntax.html) - - [Google style docstring examples](http://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). -- The documentation is set up with the ReadTheDocs Sphinx theme. - - Check out its [configuration options](https://sphinx-rtd-theme.readthedocs.io/en/latest/). -- [AutoAPI](https://sphinx-autoapi.readthedocs.io/) is used to generate documentation for the package Python objects. -- `.readthedocs.yaml` is the ReadTheDocs configuration file. When ReadTheDocs is building the documentation this package and its development dependencies are installed so the API reference can be rendered. -- [Relevant section in the guide](https://guide.esciencecenter.nl/#/best_practices/language_guides/python?id=writingdocumentation) +- Documentation should be put in the `docs/` directory. ## Coding style conventions and code quality -- [Relevant section in the NLeSC guide](https://guide.esciencecenter.nl/#/best_practices/language_guides/python?id=coding-style-conventions) and [README.dev.md](README.dev.md). - -## Continuous code quality - -[Sonarcloud](https://sonarcloud.io/) is used to perform quality analysis and code coverage report - -- `sonar-project.properties` is the SonarCloud [configuration](https://docs.sonarqube.org/latest/analysis/analysis-parameters/) file -- `.github/workflows/sonarcloud.yml` is the GitHub action workflow which performs the SonarCloud analysis +- [Relevant section in the NLeSC guide](https://guide.esciencecenter.nl/#/best_practices/language_guides/python?id=coding-style-conventions). ## Package version number diff --git a/docs/reference.md b/docs/reference.md index 4da23589..e3190807 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1,20 +1,27 @@ -Here you find the code reference for the two main components users of `PyStemmusScope` -might need. First is the StemmusScope model class. Second is the `save` module, with -which you can save the STEMMUS_SCOPE model output to a netCDF file. +Here you find the code reference for the main components of `PyStemmusScope`. -## **PyStemmusScope**: +## **Run the model**: ::: PyStemmusScope.stemmus_scope.StemmusScope options: show_root_heading: false show_source: true heading_level: 3 + members_order: source - -## **PyStemmusScope.save**: +## **Post processing**: ::: PyStemmusScope.save options: show_root_heading: true show_source: true heading_level: 3 + +## **BMI interface**: + +::: PyStemmusScope.bmi.StemmusScopeBmi + options: + show_root_heading: true + show_source: true + heading_level: 3 + members_order: source diff --git a/global_data/download_scripts/download_CAMS_CO2.py b/global_data/download_scripts/download_CAMS_CO2.py deleted file mode 100644 index 6618fb39..00000000 --- a/global_data/download_scripts/download_CAMS_CO2.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Download CAMS data using the cdsapi.""" -from pathlib import Path -import cdsapi -import certifi -import urllib3 - - -# UID+api key is read here: -with (Path.home() / ".adsloginrc").open(encoding="utf8") as f: - uid = f.readline().strip() - api_key = f.readline().strip() - -http = urllib3.PoolManager(cert_reqs="CERT_REQUIRED", ca_certs=certifi.where()) - -c = cdsapi.Client( - url="https://ads.atmosphere.copernicus.eu/api/v2", - key=f"{uid}:{api_key}", - verify=True, -) - -c.retrieve( - "cams-global-ghg-reanalysis-egg4", - { - "format": "netcdf", - "model_level": "60", # surface level data - "date": "2003-01-02/2020-12-31", - "step": ["0", "3", "6", "9", "12", "15", "18", "21"], - "variable": "carbon_dioxide", - }, - "CAMS_CO2_2003-2020.nc", -) diff --git a/global_data/download_scripts/download_FAPAR_LAI.py b/global_data/download_scripts/download_FAPAR_LAI.py deleted file mode 100644 index 018b5702..00000000 --- a/global_data/download_scripts/download_FAPAR_LAI.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Download monthly ERA5-land data using the cdsapi.""" -import itertools -from pathlib import Path -import cdsapi -from pathos.threading import ThreadPool as Pool - - -N_TRIES = 3 - - -with (Path.home() / ".cdsloginrc").open(encoding="utf8") as f: - uid = f.readline().strip() - api_key = f.readline().strip() - - -def request_lai_data(year, month): - """Request the LAI data from the CDS.""" - c = cdsapi.Client( - url="https://cds.climate.copernicus.eu/api/v2", - key=f"{uid}:{api_key}", - verify=True, - retry_max=1, - ) - - c.retrieve( - "satellite-lai-fapar", - { - "format": "zip", - "variable": "lai", - "satellite": ("proba" if year >= 2014 else "spot"), - "sensor": "vgt", - "horizontal_resolution": "1km", - "product_version": "V1", - "year": f"{year}", - "month": [ - f"{month}", - ], - "nominal_day": [ - "10", - "20", - "28", - "30", - "31", - ], - }, - f"LAI_fapar_vgt_{year}-{month}.zip", - ) - - -if __name__ == "__main__": - years = list(range(2012, 2018)) - months = [str(x).rjust(2, "0") for x in range(1, 13)] - - _years, _months = map(list, zip(*itertools.product(years, months))) - - pool = Pool(nodes=2) - pool.map(request_lai_data, _years, _months) diff --git a/global_data/download_scripts/download_cds_landcover.py b/global_data/download_scripts/download_cds_landcover.py deleted file mode 100644 index 8b0daf81..00000000 --- a/global_data/download_scripts/download_cds_landcover.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Download land cover data using the cdsapi.""" -from pathlib import Path -import cdsapi - - -with (Path.home() / ".cdsloginrc").open(encoding="utf8") as f: - uid = f.readline().strip() - api_key = f.readline().strip() - - -c = cdsapi.Client( - url="https://cds.climate.copernicus.eu/api/v2", - key=f"{uid}:{api_key}", - verify=True, -) - - -years = [2013] - - -for year in years: - c.retrieve( - "satellite-land-cover", - { - "variable": "all", - "format": "zip", - "year": f"{year}", - "version": "v2.0.7cds", - }, - f"cds_landcover_{year}.zip", - ) diff --git a/global_data/download_scripts/download_era5land_monthly.py b/global_data/download_scripts/download_era5land_monthly.py deleted file mode 100644 index 799695ae..00000000 --- a/global_data/download_scripts/download_era5land_monthly.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Download monthly ERA5-land data using the cdsapi.""" -import itertools -from pathlib import Path -import cdsapi -import certifi -import urllib3 - - -http = urllib3.PoolManager(cert_reqs="CERT_REQUIRED", ca_certs=certifi.where()) - -with (Path.home() / ".cdsloginrc").open(encoding="utf8") as f: - uid = f.readline().strip() - api_key = f.readline().strip() - -c = cdsapi.Client( - url="https://cds.climate.copernicus.eu/api/v2", - key=f"{uid}:{api_key}", - verify=True, -) - -variables = ["2m_dewpoint_temperature", "2m_temperature"] -years = [2014] -months = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"] - -for var, year, month in itertools.product(variables, years, months): - # fmt: off - c.retrieve( - "reanalysis-era5-land", - { - "variable": [var], - "year": f"{year}", - "month": month, - "day": [ - "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", - "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", - "23", "24", "25", "26", "27", "28", "29", "30", "31", - ], - "time": [ - "00:00", "01:00", "02:00", "03:00", "04:00", "05:00", "06:00", - "07:00", "08:00", "09:00", "10:00", "11:00", "12:00", "13:00", - "14:00", "15:00", "16:00", "17:00", "18:00", "19:00", "20:00", - "21:00", "22:00", "23:00", - ], - "area": [65, 20, 60, 24], - "format": "netcdf", - }, - f"era5-land_{var}_{year}-{month}_FI-Hyy.nc", - ) - # fmt: on diff --git a/global_data/downloading_global_data.md b/global_data/downloading_global_data.md deleted file mode 100644 index 436dba06..00000000 --- a/global_data/downloading_global_data.md +++ /dev/null @@ -1,83 +0,0 @@ -# Global data for STEMMUS_SCOPE -This document outlines which, where and how we download the "global" input data for the -model. - -## Data downloaded through era5cli -Many of the forcing-related data is available in the era5 reanalysis data. - -**era5 variables**: - - mean_total_precipitation_rate - - surface_thermal_radiation_downwards - - surface_solar_radiation_downwards - - surface_pressure - - 10m_u_component_of_wind - - 10m_v_component_of_wind - -These can be downloaded using [era5cli](https://era5cli.readthedocs.io/) as: -``` -era5cli hourly --variables 10m_u_component_of_wind 10m_v_component_of_wind mean_total_precipitation_rate surface_pressure surface_thermal_radiation_downwards surface_solar_radiation_downwards --startyear 2016 --endyear 2016 --levels surface --area 65 20 60 24 -``` - -Note: the year range is set with `--startyear` and `--endyear`. The lat/lon bounding box is set with `--area`. - -**era5-land variables**: - - 2m_temperature - - 2m_dewpoint_temperature - -We would like to run the following command: -``` -era5cli hourly --variables 2m_temperature 2m_dewpoint_temperature --startyear 2016 --endyear 2016 --months 1 --land --levels surface --area 54 3 50 8 -``` -However, currently this raises a too-many-requests error. Until that is fixed, we can use -the cds. See `download_era5land_monthly.py`. - -**era5-land soil initial conditions** - -For running STEMMUS-SCOPE, global data is also required for the soil initial conditions. These are retrieved from ERA5-land, using the following command: -``` -era5cli hourly --startyear 2014 --endyear 2014 --hours 0 --land --levels surface --area 65 20 60 24 --variables skin_temperature soil_temperature_level_1 soil_temperature_level_2 soil_temperature_level_3 soil_temperature_level_4 volumetric_soil_water_layer_1 volumetric_soil_water_layer_2 volumetric_soil_water_layer_3 volumetric_soil_water_layer_4 -``` - -## CO2 data -CO2 data is available in the CAMS dataset. An ADS script that can download the data is - available at `download_scripts/download_CAMS_CO2.py`. - -A simple check for the parsing of the data is in `data_analysis_notebooks/parse_CO2_data.ipynb`. - - -## Canopy height data -The canopy height data is described in: https://langnico.github.io/globalcanopyheight/ - -Get data from: -https://share.phys.ethz.ch/~pf/nlangdata/ETH_GlobalCanopyHeight_10m_2020_version1/3deg_cogs/ -This can be done with, e.g. a wget command. - -The valid filenames are all in `download_scripts/valid_eth_canopy_height_files.txt`. - -A simple example for the parsing of the data is in `data_analysis_notebooks/parse_canopy_height.ipynb`. - - -## DEM data -To download the DEM data: -`wget https://prism-dem-open.copernicus.eu/pd-desk-open-access/prismDownload/COP-DEM_GLO-90-DGED__2021_1/Copernicus_DSM_30_N35_00_E012_00.tar` -unzip and extract tif file. - -All valid DEM urls are in `download_scripts/valid_dem_urls.csv`. - -A word doc for instructions is available [here](https://spacedata.copernicus.eu/documents/20123/121286/Copernicus+DEM+Open+HTTPS+Access.pdf/36c9adad-8488-f463-af43-573e68b7f481?t=1669283200177) - -A simple example for the parsing of the data is in `data_analysis_notebooks/parse_dem.ipynb`. - - -## LAI - -LAI data was retrieved from the CDS. However, there are some downloading issues with -the `satellite-lai-fapar` dataset. A ticket has been opened at the ECMWF. - -The download script for downloading the LAI data is available under `download_scripts/download_FAPAR_LAI.py`. - -A simple example for parsing the LAI data is in `data_analysis_notebooks/parse_LAI.py`. - -## Land cover - -Land cover data is currently not implemented yet either. diff --git a/mkdocs.yml b/mkdocs.yml index a31e3fb9..574c24a2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,8 @@ repo_url: https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing repo_name: PyStemmusScope nav: - - Getting started: index.md + - Introduction: index.md + - Getting started: getting_started.md - Installation: installation_instructions.md - Run the model: - Different ways to run the model: notebooks/run_model_on_different_infra.ipynb @@ -11,10 +12,10 @@ nav: - Site and global datasets on CRIB: notebooks/run_model_with_different_dataset_on_CRIB.ipynb - BMI: - - "BMI instructions": bmi.md - - "BMI demonstration": notebooks/BMI_demo.ipynb - - "grpc4bmi demonstration": notebooks/grpc4bmi_demo.ipynb - - "example: coupling modflow and stemmus_scope": notebooks/bmi_MODFLOW_coupling.ipynb + - BMI instructions: bmi.md + - BMI demonstration: notebooks/BMI_demo.ipynb + - grpc4bmi demonstration: notebooks/grpc4bmi_demo.ipynb + - example-coupling modflow and stemmus_scope: notebooks/bmi_MODFLOW_coupling.ipynb - Contributing guide: CONTRIBUTING.md - API reference: reference.md diff --git a/pull_request_template.md b/pull_request_template.md new file mode 100644 index 00000000..1fe3abc6 --- /dev/null +++ b/pull_request_template.md @@ -0,0 +1,19 @@ +### Description + +Please add a description of the changes proposed in the pull request. + +### Checklist + +- [ ] Add a reference to related issues. +- [ ] @mentions of the person or team responsible for reviewing proposed changes. +- [ ] This pull request has a descriptive title. +- [ ] Follow [contributing guide](https://pystemmusscope.readthedocs.io/en/latest/CONTRIBUTING/). +- [ ] The checks, below the pull request, are successful (green). +- [ ] Add documentation. +- [ ] Add tests. +- [ ] Add changes to the [changelog file](./docs/CHANGELOG.md) under section `Unreleased`. +- [ ] If your changes are about BMI, make sure that your changes are compatible + with [the latest version of + STEMMUS_SCOPE](https://github.com/EcoExtreML/STEMMUS_SCOPE/releases) and [its + of docker + image](https://github.com/orgs/EcoExtreML/packages?repo_name=STEMMUS_SCOPE).