From 20f60eaa57baa0f9b97eb124863e9a8d9759e1c2 Mon Sep 17 00:00:00 2001 From: Espen Hagen <2492641+espenhgn@users.noreply.github.com> Date: Mon, 27 May 2024 16:26:01 +0200 Subject: [PATCH] build docker image/sif file action Fixes #6 --- .github/workflows/container_build_push.yml | 87 +++++++++++++++++ README.md | 3 +- docker/README.md | 6 +- .../dockerfiles/container_template/Dockerfile | 2 +- docker/scripts/apt_get_essential.sh | 6 +- docker/scripts/install_mambaforge.sh | 2 +- docs/container_template.md | 2 +- scripts/PROJECT_README.md | 95 ++++++++++++++++++- version/version.py | 2 +- 9 files changed, 192 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/container_build_push.yml diff --git a/.github/workflows/container_build_push.yml b/.github/workflows/container_build_push.yml new file mode 100644 index 0000000..1c4b683 --- /dev/null +++ b/.github/workflows/container_build_push.yml @@ -0,0 +1,87 @@ +name: container-build-push + +on: + push: + branches: + - 'main' + tags: + - "v*.*.*" + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + ghcr.io/${{ github.repository_owner }}/container_template + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha + type=raw,value=latest,enable={{is_default_branch}} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:src" + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + file: + ./dockerfiles/container_template/Dockerfile + platforms: linux/amd64 + + build-apptainer-container: + needs: docker + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + name: Build Apptainer Container + steps: + - name: Check out code for the container builds + uses: actions/checkout@v4 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/${{ github.repository_owner }}/container_template + tags: | + type=semver,pattern={{version}} + type=ref,event=branch + type=sha + type=raw,value=latest,enable={{is_default_branch}} + - name: Build and push Apptainer container + run: | + tags="${{ steps.meta.outputs.tags }}" + csv_tags=$(printf "%s\n" "$tags" | awk -F: 'NR==1{printf "%s,", $0; next} {printf "%s,", $NF}' | sed 's/,$//') + IFS= read -r first_tag < ... @@ -27,6 +27,7 @@ Revise the `/.gitattributes` file as necessary. Some common [![Documentation Status](https://readthedocs.org/projects/container-template/badge/?version=latest)](https://container-template.readthedocs.io/en/latest/?badge=latest) [![Flake8 lint](https://github.com/precimed/container_template/actions/workflows/python.yml/badge.svg)](https://github.com/precimed/container_template/actions/workflows/python.yml) [![Dockerfile lint](https://github.com/precimed/container_template/actions/workflows/docker.yml/badge.svg)](https://github.com/precimed/container_template/actions/workflows/docker.yml) +[![Container build push](https://github.com/precimed/container_template/actions/workflows/container_build_push.yml/badge.svg)](https://github.com/precimed/container_template/actions/workflows/container_build_push.yml) ## Description of available containers @@ -36,7 +37,92 @@ Revise the `/.gitattributes` file as necessary. Some common Below is the list of tools included in the different Dockerfile(s) and installer bash scripts for each container. Please keep up to date (and update the main `/README.md` when pushing new container builds): - + +### Installation and set up + +#### Dependencies on host system + +In order to set up these resource, some software may be required + +- [Singularity/SingularityCE](https://sylabs.io/singularity/) or [Apptainer](https://apptainer.org) +- [Git](https://git-scm.com/) +- [Git LFS](https://git-lfs.com) +- [ORAS CLI](https://oras.land) + +#### Clone the repository + +To download the last revision of this project, issue: + +```bash +cd path/to/repositories +git clone --depth 1 https://github.com/precimed/container_template.git +cd container_template +git lfs pull # pull "large" files +``` + +#### Update the `container_template.sif` container + +To obtain updated versions of the Singularity Image Format (.sif) container file `, issue + +```bash +cd path/to/repositories/container_template/singularity +mv container_template.sif container_template.sif.old # optional, just rename the old(er) file +apptainer pull docker://ghcr.io/precimed/container_template: # or +singularity pull docker://ghcr.io/precimed/container_template: # or +oras pull ghcr.io/precimed/container_template_sif: +``` + +where `` corresponds to a tag listed under [packages](https://github.com/precimed/container_template/pkgs/container/container_template), +such as `latest`, `main`, or `sha_`. +The `oras pull` statement pulls the `container_template.sif` file from [ghcr.io](https://github.com/precimed/container_template/pkgs/container/container_template_sif) using the [ORAS](https://oras.land) registry, without the need to build the container locally. + +#### Pulling and using Docker image + +To pull the corresponding Docker image, issue: + +```bash +docker pull ghcr.io/precimed/container_template: +``` + +If working on recent Macs, add the `--platform=linux/amd64` after `docker pull`. +This may allow replacing `singularity exec ...` or `apptainer exec ...` statements with appropriate `docker run ...` statements, +on systems where Singularity or Apptainer is unavailable. +Functionally, the Docker image is equivalent to the Singularity container, but note that syntax for mounting volumes and invoking commands may differ. +Please refer to [docs.docker.com](https://docs.docker.com) for more information. + +> [!NOTE] Note that the provided Docker image may not support all CPUs, and may not be able to run on all systems via CPU virtualization. +> An option may be to build the Docker image on the host machine (e.g., M1/M2 Macs, older Intel CPUs), as: +> +>```bash +>docker build --platform=linux/amd64 -t ghcr.io/precimed/container_template -f dockerfiles/container_template/Dockerfile . +>``` + +Example of using the Docker image: + +```bash +#!/bin/bash +# define environment variables: +export IMAGE="ghcr.io/precimed/container_template:latest" # adapt as necessary +# shortcuts for Python and interactive shell: +export PYTHON="docker run --platform=linux/amd64 --rm -v ${PWD}:/home -w/home --entrypoint=python ${IMAGE}" +export ISHELL="docker run --platform=linux/amd64 --rm -it -v ${PWD}:/home -w/home --entrypoint=bash ${IMAGE}" + +# invoke Python help/list local directory +$PYTHON --help +$PYTHON -c "import os; print(os.listdir())" +``` + +### Systems without internet access + +Some secure platforms do not have direct internet access, hence we recommend cloning/pulling all required files on a machine with internet access as explained above, and archive the `container_template` directory with all files and moving it using whatever file uploader is available for the platform. + +```bash +cd /path/to/container_template +SHA=$(git rev-parse --short HEAD) +cd .. +tar --exclude=".git/*" -cvf container_template_$SHA.tar container_template +``` + ### container_template.sif | OS/tool | Version | License | Source @@ -46,12 +132,13 @@ Please keep up to date (and update the main `/README.md` whe ## Building/rebuilding containers -For instructions on how to build or rebuild containers using [Docker](https://www.docker.com) and [Singularity](https://docs.sylabs.io) refer to [`/docker/README.md`](https://github.com/precimed/container_template/blob/main/docker/README.md). +While we don't recommend building containers locally, it is possible. +For instructions on how to build or rebuild containers manually using [Docker](https://www.docker.com) and [Singularity](https://docs.sylabs.io) refer to [`/docker/README.md`](https://github.com/precimed/container_template/blob/main/docker/README.md). ## Build the documentation Within this repository, the html-documentation can be built from source files put here using [Sphinx](https://www.sphinx-doc.org/en/master/index.html). -To do so, install Sphinx and some additional packages in python using [Conda](https://docs.conda.io/en/latest/) by issuing: +To do so, install Sphinx and some additional packages in Python using [Conda](https://docs.conda.io/en/latest/) by issuing: ``` cd /docs/source diff --git a/version/version.py b/version/version.py index 34d4cc8..7b2e674 100644 --- a/version/version.py +++ b/version/version.py @@ -5,7 +5,7 @@ _PATCH = "0" # This is mainly for nightly builds which have the suffix ".dev$DATE". See # https://semver.org/#is-v123-a-semantic-version for the semantics. -_SUFFIX = "rc3" +_SUFFIX = "rc4" VERSION_SHORT = "{0}.{1}".format(_MAJOR, _MINOR) VERSION = "{0}.{1}.{2}{3}".format(_MAJOR, _MINOR, _PATCH, _SUFFIX) \ No newline at end of file