From 9f527e93cfb237e322cf88cf3d2c87ac28f923fd Mon Sep 17 00:00:00 2001 From: Henrik Finsberg Date: Thu, 16 Nov 2023 09:31:59 +0100 Subject: [PATCH 1/7] Write about modules and packages --- cspell.json | 2 ++ docs/distribution.md | 31 +++++++++++++++++++++++++ docs/environments.md | 55 +++++++++++++++++++++++++++++++++++++------- docs/testing.md | 11 --------- 4 files changed, 80 insertions(+), 19 deletions(-) diff --git a/cspell.json b/cspell.json index 2cbb9bd..269fe1e 100644 --- a/cspell.json +++ b/cspell.json @@ -19,6 +19,8 @@ "Finsberg", "ipython", "isort", + "itertools", + "itools", "Jørgen", "jupytext", "linkify", diff --git a/docs/distribution.md b/docs/distribution.md index fbf3d80..def8bda 100644 --- a/docs/distribution.md +++ b/docs/distribution.md @@ -29,3 +29,34 @@ Henrik Finsberg and Jørgen Dokken - Creating a GitHub action to publish a new docker image to the GitHub registry whenever you make a new release - Licensing - Which license to put on your code and why you should use a permissive license + +--- + +## Creating a package + +- A set of modules can be collected in a package + +- A package is organized as module files in a directory tree + +- Each subdirectory must have a __init__.py file (can be empty) + +``` +examples/my-package +├── LICENSE +├── pyproject.toml +├── README.md +├── src +│ └── pkg +│ ├── analysis.py +│ ├── __init__.py +│ └── printing +│ ├── __init__.py +│ └── printing.py +└── test + ├── test_analysis.py + └── test_printing.py +``` + +--- + +## Installing a package diff --git a/docs/environments.md b/docs/environments.md index 3840221..fe504d5 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -19,14 +19,53 @@ Henrik Finsberg and Jørgen Dokken --- -## Agenda - -- What is a development environment -- Virtual environments in python - - pinning dependencies with `pip-tools` - - pyproject.toml -- Conda -- Docker +- A module is a file consisting of Python code +- A package is a hierarchical file directory structure that consists of modules and sub-packages + +![bg fit right 40%](https://uio-in3110.github.io/_images/python_structure_options.svg) + +--- + +## Using modules + +```python +import itertools +# Access function from the module +itertools.product + +# Alias +import itertools as itools +itools.product + +# The following is considered a bad practice +from itertools import * +# Easy to shadow existing variables (also hard for IDEs) +``` + + +--- + +## Using packages + +```python +from scipy.optimize import minimize +# ^ ^ ^ +# | | | +# Package | | +# Module | +# Function +``` + +--- + +## How does python know which modules and packages that are available? + +```python +import sys + +# Notice the order +sys.path +``` --- diff --git a/docs/testing.md b/docs/testing.md index ef35336..b0c9475 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -20,17 +20,6 @@ Henrik Finsberg and Jørgen Dokken --- -## Agenda - -- Linters -- Formatters -- pre-commit hooks -- Running pre-commit hooks in using GitHub actions - - - ---- - ## What is a linter? - A linter is a tool that analyses your source code and reports potential problems. From fad1650bc8898e7ffec4db25e690e034cbf93629 Mon Sep 17 00:00:00 2001 From: Henrik Finsberg Date: Thu, 16 Nov 2023 11:36:56 +0100 Subject: [PATCH 2/7] More about file structure and readme file --- cspell.json | 1 + docs/documentation.md | 36 ++++++++++++------ docs/environments.md | 2 +- docs/repo.md | 86 +++++++++++++++++++++++++++++-------------- 4 files changed, 85 insertions(+), 40 deletions(-) diff --git a/cspell.json b/cspell.json index 269fe1e..484225e 100644 --- a/cspell.json +++ b/cspell.json @@ -23,6 +23,7 @@ "itools", "Jørgen", "jupytext", + "licence", "linkify", "marp", "matplotlib", diff --git a/docs/documentation.md b/docs/documentation.md index 51ebdf6..79ed37f 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -18,17 +18,6 @@ Best Practices in Modern Software Development: 23.11.23 Henrik Finsberg and Jørgen Dokken ---- - -## Agenda - -- Why documentation? -- Writing docstrings and generating API documentation - - numpy vs Google style -- Writing documentation using Jupyter-Book -- Hosting documentation on GitHub pages - - --- ## What is documentation? @@ -53,6 +42,31 @@ Henrik Finsberg and Jørgen Dokken --- +## The README file + +- The first documentation a user reads is the README file +- Should include + - Title of the project + - Description of the project + - Installation instructions + - How to get started + +--- + +- Could include + - Badges + - Information about how to contribute + - License information + - Credits + - Example + - How to cite + - Screenshots / figures +- https://readme.so + + +--- + + ## Using JupyterBook to create documentation - [JupyterBook](https://jupyterbook.org/en/stable/intro.html) is a powerful framework for writing documentation diff --git a/docs/environments.md b/docs/environments.md index fe504d5..033c8cd 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -22,7 +22,7 @@ Henrik Finsberg and Jørgen Dokken - A module is a file consisting of Python code - A package is a hierarchical file directory structure that consists of modules and sub-packages -![bg fit right 40%](https://uio-in3110.github.io/_images/python_structure_options.svg) +![bg fit right](https://uio-in3110.github.io/_images/python_structure_options.svg) --- diff --git a/docs/repo.md b/docs/repo.md index 6c19a9b..55f373e 100644 --- a/docs/repo.md +++ b/docs/repo.md @@ -17,31 +17,6 @@ Best Practices in Modern Software Development: 23.11.23 Henrik Finsberg and Jørgen Dokken ---- - -## Agenda - -- Why version control systems - - Why do we need version control systems. - - Why it is good to track the history of a project and understand why certain decisions where made -- Setting up your first repository - - Cookiecutters and template repositories -- What is GitHub? - - Collaboration with other through code review - - Pull requests process - - Issue tracking - ---- - -- Releases and tags - - Create a release with a tags for specific version that you want to easily be able to go back to, e.g when you submit a paper or when a paper is published. - - Creating a changelog to show what has been changed between releases. -- Licenses -- Data repositories and data sharing - - How to share data. This should typically not be in a git repo, but rather on a data repository such as Zenodo. - - How to deal with the case when it is not possible to share data (Generate synthetic data). - - --- ## Why version control systems? @@ -50,6 +25,9 @@ Henrik Finsberg and Jørgen Dokken * To make it easy to go back to a previous version * To make changes while maintaining a working version +![bg fit right](https://the-turing-way.netlify.app/_images/project-history.svg) +

Image is used under a CC-BY 4.0 license. DOI: 10.5281/zenodo.3332807.

+ --- ## Setting up your first project @@ -61,6 +39,8 @@ Henrik Finsberg and Jørgen Dokken - https://www.cookiecutter.io/templates - Templates - Copy all files from an existing repository + - https://github.com/scientificcomputing/example-paper + - https://github.com/scientificcomputing/example-paper-fenics --- @@ -74,12 +54,43 @@ cookiecutter gh:scientificcomputing/generate-paper --- +``` +research_paper_1 +├── CITATION.cff # Info about how to cite your project +├── LICENSE # The license +├── README.md # What the user should read first +├── _config.yml # Configurations for docs +├── _toc.yml # Table of contents for docs +├── code # Where to put your code +│ └── README.md # Description of the code +├── cspell.config.yaml # Dictionary for spell checker +├── data # Where to put your data +│ └── README.md # Description of the data +├── docker +│ └── Dockerfile # The docker file +├── docs # Where to put your docs +│ ├── logo.png # Simula Logo to put in documentation +│ └── references.bib # Where to put your references +├── environment.yml # Conda dependencies +└── pyproject.toml # Python metadata and dependencies +``` + +--- + ## What is GitHub? * A place to host your remote repositories * To make it easier to collaborate with others * To keep a backup (on GitHub) -* Create repositories under your group's GitHub organization +* Create repositories under your group's GitHub organization (this will make it easier if you are unavailable) + - https://github.com/scientificcomputing + - https://github.com/ComputationalPhysiology + +--- + +![bg w:50%](https://the-turing-way.netlify.app/_images/healthy-research-tree.jpg) + +

Image is used under a CC-BY 4.0 licence. DOI: 10.5281/zenodo.3332807.

--- @@ -94,13 +105,14 @@ cookiecutter gh:scientificcomputing/generate-paper --- # Demo + - Create a new branch and open a pull request --- # Exercise -- Create a new repo on GitHub (ex: ) +- Create a new repo on GitHub (ex: `example-paper`) - Run the cookiecutter and add all the files in a single commit ``` git init # Initialize @@ -137,7 +149,7 @@ Find a typo, mistake or a missing feature in any of the repositories under Date: Thu, 16 Nov 2023 11:55:04 +0100 Subject: [PATCH 3/7] Fix typos --- cspell.json | 1 + docs/environments.md | 11 ++++------- docs/repo.md | 6 +++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cspell.json b/cspell.json index 484225e..d2e11de 100644 --- a/cspell.json +++ b/cspell.json @@ -37,6 +37,7 @@ "pyproject", "pytest", "pyupgrade", + "Ragan", "repr", "scientificcomputing", "scipy", diff --git a/docs/environments.md b/docs/environments.md index 033c8cd..1c170d0 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -15,7 +15,7 @@ marp: true # Reproducible environments Best Practices in Modern Software Development: 23.11.23 -Henrik Finsberg and Jørgen Dokken +Henrik Finsberg, Jørgen Dokken and Benjamin Ragan-Kelly --- @@ -155,8 +155,6 @@ deactivate ## Example `pyproject.toml` -FIXME: No we need build-system? - ```toml [build-system] # Setuptools + editable install requires = ["setuptools>=61.2"] @@ -268,6 +266,9 @@ pip-compile --extra=dev --output-file=requirements-dev.txt pyproject.toml --- +Demo + +--- ## Conda @@ -275,10 +276,6 @@ TBW --- -Demo - ---- - ## Docker [Docker](https://www.docker.com/get-started/) is a platform that packages an application and all its dependencies together in the form of containers. diff --git a/docs/repo.md b/docs/repo.md index 55f373e..af3b43d 100644 --- a/docs/repo.md +++ b/docs/repo.md @@ -223,7 +223,7 @@ https://semver.org ## Licenses -- What can other users to with the material in your repository? +- What can other users do with the material in your repository? - No license means the nobody can use, copy, distribute, or modify the work without consent from the author * Add a file called LICENSE to your repository. Go to GitHub, click "Add file" and type the name `LICENSE` and GitHub will provide you with some options @@ -234,7 +234,7 @@ https://semver.org - MIT: Permissive - Others can use your code in any way, and you will not be sued if the software doesn't work (recommended in most cases) - GPL: Copyleft - derivative work must use the same license - good way to embrace open source but often problematic for commercial companies - LGPL: Similar to GPL but software can be used under different license -- CC-BY-4.0 - Typically used for creative work (more journals use this) +- CC-BY-4.0 - Typically used for creative work (most journals use this) https://choosealicense.com @@ -280,7 +280,7 @@ Example: https://github.com/ComputationalPhysiology/mps - Git does not work well with binary files - Instead you should store large files in a data repository - Use Google Drive / Dropbox / Other while developing - - Publish Data on Zenodo when ready + - Publish Data on Zenodo when ready (Zenodo and GitHub integrates well) --- From 4abf5dc411e338b010c55bb2b9c97513b0b18383 Mon Sep 17 00:00:00 2001 From: Henrik Finsberg Date: Thu, 16 Nov 2023 12:25:47 +0100 Subject: [PATCH 4/7] Add linting before docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f1faf6..49de631 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ The agenda is as follows - Code and Data repositories [Slides](https://scientificcomputing.github.io/seminar-23-11-2023/repo-slides.html) [Book](https://scientificcomputing.github.io/seminar-23-11-2023/docs/repo.html) - Reproducible environments [Slides](https://scientificcomputing.github.io/seminar-23-11-2023/environments-slides.html) [Book](https://scientificcomputing.github.io/seminar-23-11-2023/docs/environments.html) -- Documentation [Slides](https://scientificcomputing.github.io/seminar-23-11-2023/documentation-slides.html) [Book](https://scientificcomputing.github.io/seminar-23-11-2023/docs/documentation.html) - Linters, formatters and continuous integration [Slides](https://scientificcomputing.github.io/seminar-23-11-2023/testing-slides.html) [Book](https://scientificcomputing.github.io/seminar-23-11-2023/docs/testing.html) +- Documentation [Slides](https://scientificcomputing.github.io/seminar-23-11-2023/documentation-slides.html) [Book](https://scientificcomputing.github.io/seminar-23-11-2023/docs/documentation.html) - Example paper with code [Slides](https://scientificcomputing.github.io/seminar-23-11-2023/paper-slides.html) [Book](https://scientificcomputing.github.io/seminar-23-11-2023/docs/paper.html) - Packaging and Distribution (if time) [Slides](https://scientificcomputing.github.io/seminar-23-11-2023/distribution-slides.html) [Book](https://scientificcomputing.github.io/seminar-23-11-2023/docs/distribution.html) From 95037c765a3eb7d9bb1d343bd53d53c6e8d452f6 Mon Sep 17 00:00:00 2001 From: Henrik Finsberg Date: Thu, 16 Nov 2023 12:26:42 +0100 Subject: [PATCH 5/7] Add placeholder for CI --- docs/testing.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/testing.md b/docs/testing.md index b0c9475..ed6fdb0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -218,3 +218,12 @@ jobs: - uses: actions/setup-python@v4 - uses: pre-commit/action@v3.0.0 ``` + +--- + +## Continuous Integration + + +TBW + +--- From 5c79b6c8d37929c56448d9742391e56a1638f12a Mon Sep 17 00:00:00 2001 From: Henrik Finsberg Date: Thu, 16 Nov 2023 12:44:23 +0100 Subject: [PATCH 6/7] CI for docs --- docs/documentation.md | 117 ++++++++++++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/docs/documentation.md b/docs/documentation.md index 79ed37f..67f9b12 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -24,8 +24,7 @@ Henrik Finsberg and Jørgen Dokken * Background and motivation - What do you try to solve? -* Instructions on how to install the software - - Installation instructions +* Instructions on how to install the software and necessary dependencies * Instructions on how to use the software - Tutorials / demos - API documentation @@ -249,14 +248,78 @@ parse: ## Publishing the book to GitHub pages -- Create a new file `.github/workflows/build_docs.yml` +- Create a new file `.github/workflows/build_docs.yml` which contains a workflow for building the docs +- Also remember to go to your [repository settings on GitHub](https://docs.github.com/en/pages/getting-started-with-github-pages/configuring-a-publishing-source-for-your-github-pages-site#publishing-with-a-custom-github-actions-workflow) to allow for GitHub pages, see + +--- ```yaml # Simple workflow for deploying static content to GitHub Pages -name: Deploy static content to Pages +name: Build docs + +on: + workflow_dispatch: + workflow_call: + pull_request: + branches: + - main + +jobs: + build_docs: + runs-on: ubuntu-22.04 + env: + PYTHON_VERSION: "3.10" + PUBLISH_DIR: ./_build/html + + steps: + # checkout the repository + - uses: actions/checkout@v4 + + # setup Python + - name: Install Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} -on: [push] + # preserve pip cache to speed up installation + - name: Cache pip + uses: actions/cache@v3 + with: + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ hashFiles('*requirements-docs.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements-docs.txt + + - name: Build docs + run: python3 -m jupyter book build . + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + name: documentation + path: ${{ env.PUBLISH_DIR }} + if-no-files-found: error +``` + +--- + +## Publishing the book to GitHub pages + +- Create a new file `.github/workflows/publish_docs.yml` which contains a workflow for publishing the docs to github pages + +--- + +```yaml +name: Github Pages + +on: + push: + branches: [main] # Only run on push to main # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: @@ -269,49 +332,37 @@ concurrency: group: "pages" cancel-in-progress: true -env: - # Directory that will be published on github pages - PUBLISH_DIR: ./_build/html - jobs: + build-docs: + uses: ./.github/workflows/build_docs.yml - build: - runs-on: ubuntu-22.04 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install dependencies - run: | - python3 -m pip install jupyter-book jupytext - - - name: Build docs - run: jupyter book build . - - - name: Upload artifact - uses: actions/upload-pages-artifact@v2 - with: - path: ${{ env.PUBLISH_DIR }} - - # Single deploy job since we're just deploying deploy: - if: github.ref == 'refs/heads/main' - needs: build + needs: [build-docs] + environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest - steps: + - name: Download docs artifact + # docs artifact is uploaded by build-docs job + uses: actions/download-artifact@v3 + with: + name: documentation + path: "./public" + + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: "./public" + - name: Checkout uses: actions/checkout@v4 - name: Setup Pages uses: actions/configure-pages@v3 - - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v2 From 511c79e2310659767f7b303ecb5dd710da2fffd4 Mon Sep 17 00:00:00 2001 From: Henrik Finsberg Date: Thu, 16 Nov 2023 12:46:08 +0100 Subject: [PATCH 7/7] Fix errors with transistion --- docs/environments.md | 2 -- docs/testing.md | 2 -- 2 files changed, 4 deletions(-) diff --git a/docs/environments.md b/docs/environments.md index 1c170d0..fed3aaf 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -409,5 +409,3 @@ The simplest way to ensure that users than exactly reproduce your environment is Therefore you should always publish a docker image containing the exact dependencies for reproducing the results The templates contains a workflow for building docker images using GitHub actions for every new tag - ---- diff --git a/docs/testing.md b/docs/testing.md index ed6fdb0..470ede9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -225,5 +225,3 @@ jobs: TBW - ----