From 29c249b942641698848da094ee8b9d28b8b490e0 Mon Sep 17 00:00:00 2001 From: Elisabetta Iavarone <18575092+elisabettai@users.noreply.github.com> Date: Fri, 9 Feb 2024 11:40:12 +0100 Subject: [PATCH] First version working locally (with run-local) --- .bumpversion.cfg | 17 ++ .cookiecutterrc | 41 ++++ .github/CODEOWNERS | 15 ++ .github/ISSUE_TEMPLATE/ask_question.md | 14 ++ .github/ISSUE_TEMPLATE/bug_report.md | 43 ++++ .github/ISSUE_TEMPLATE/feature_request.md | 27 +++ .github/workflows/check-image.yml | 22 ++ .osparc/docker-compose.overwrite.yml | 5 + .osparc/metadata.yml | 197 +++++++++++++++++ .osparc/runtime.yml | 24 ++ CHANGELOG.md | 5 + Dockerfile | 241 +++++++++++++++++++++ Makefile | 127 +++++++++++ README.ipynb | 61 ++++++ README.md | 27 +++ boot_scripts/boot_notebook.bash | 87 ++++++++ boot_scripts/entrypoint.bash | 77 +++++++ docker-compose-local.yml | 16 ++ original_jupyter_scripts/fix-permissions | 33 +++ original_jupyter_scripts/initial-condarc | 6 + original_jupyter_scripts/start-notebook.py | 44 ++++ original_jupyter_scripts/start-notebook.sh | 5 + validation/inputs/.gitkeep | 0 validation/outputs/.gitkeep | 0 validation/workspace/.gitkeep | 0 25 files changed, 1134 insertions(+) create mode 100644 .bumpversion.cfg create mode 100644 .cookiecutterrc create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/ask_question.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/workflows/check-image.yml create mode 100644 .osparc/docker-compose.overwrite.yml create mode 100644 .osparc/metadata.yml create mode 100644 .osparc/runtime.yml create mode 100644 CHANGELOG.md create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.ipynb create mode 100644 README.md create mode 100755 boot_scripts/boot_notebook.bash create mode 100755 boot_scripts/entrypoint.bash create mode 100644 docker-compose-local.yml create mode 100644 original_jupyter_scripts/fix-permissions create mode 100644 original_jupyter_scripts/initial-condarc create mode 100644 original_jupyter_scripts/start-notebook.py create mode 100644 original_jupyter_scripts/start-notebook.sh create mode 100644 validation/inputs/.gitkeep create mode 100644 validation/outputs/.gitkeep create mode 100644 validation/workspace/.gitkeep diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..0a82ba1 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,17 @@ +[bumpversion] +current_version = 1.0.0 +commit = False +message = service version: {current_version} → {new_version} +tag = False + +[bumpversion:file:.osparc/metadata.yml] +search = {current_version} +replace = {new_version} + +[bumpversion:file:Makefile] +search = {current_version} +replace = {new_version} + +[bumpversion:file:docker-compose-local.yml] +search = {current_version} +replace = {new_version} diff --git a/.cookiecutterrc b/.cookiecutterrc new file mode 100644 index 0000000..25917b0 --- /dev/null +++ b/.cookiecutterrc @@ -0,0 +1,41 @@ +# This file exists so you can easily regenerate your project. +# +# `cookiepatcher` is a convenient shim around `cookiecutter` +# for regenerating projects (it will generate a .cookiecutterrc +# automatically for any template). To use it: +# +# pip install cookiepatcher +# cookiepatcher gh:itisfoundation/cookiecutter-osparc-service project-path +# +# See: +# https://pypi.python.org/pypi/cookiepatcher +# +# Alternatively, you can run: +# +# cookiecutter --overwrite-if-exists --config-file=project-path/.cookiecutterrc gh:itisfoundation/cookiecutter-osparc-service +# + +default_context: + + _output_dir: '/home/iavarone/tests' + _template: '/home/iavarone/repos/cookiecutter-osparc-jupyterlab-service/' + author_affiliation: "IT'IS Foundation" + author_email: 'iavarone@itis.swiss' + author_name: 'Elisabetta Iavarone' + contact_email: 'iavarone@itis.swiss' + docker_base: 'quay.io/jupyter/minimal-notebook:7285848c0a11@sha256:1f43611f561bd6e8480f4bc03d0c56ce56b6657f1b21cd2bfb2debbb54973164' + git_repo: 'github' + git_username: 'elisabettai' + install_custom_software: 'no' + install_other_kernel: 'no' + install_python_kernel: 'yes' + number_of_inputs: '4' + number_of_outputs: '4' + project_name: 'AxonDeepSeg in JupyterLab' + project_package_name: 'jupyter_axondeepseg' + project_short_description: 'AxonDeepSeg in JupyterLab' + project_slug: 'jupyter-axondeepseg' + release_date: '2024' + resources_CPU_nanoCPUs: '4000000000' + resources_RAM_nanoBytes: '16000000000' + version: '1.0.0' diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..ca4e0da --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,15 @@ + +# Maps code in repository with maintainers +# Order is important. The last matching pattern has the most precedence. +# SEE https://docs.github.com/en/free-pro-team@latest/github/creating-cloning-and-archiving-repositories/about-code-owners#example-of-a-codeowners-file + + +# files and folders recursively +Dockerfile @elisabettai +Makefile @elisabettai + +# NOTE: '/' denotes the root of the repository +/.github/ @pcrespov @GitHK +/.osparc/ @pcrespov @GitHK @elisabettai +/kernels/ @elisabettai +/docker/ @pcrespov @GitHK diff --git a/.github/ISSUE_TEMPLATE/ask_question.md b/.github/ISSUE_TEMPLATE/ask_question.md new file mode 100644 index 0000000..361e55b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/ask_question.md @@ -0,0 +1,14 @@ +--- +name: 💬 Question +about: Ask a question +labels: question +--- + +## What version of this service are you using? + + + +## How can we help you? diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..9c58e32 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,43 @@ +--- +name: 🐛 Bug +about: File a bug/issue +labels: bug +--- + +## What version of this service are you using? + + + + + +## Long story short + + + +## Expected behaviour + + + +## Actual behaviour + + + +## Steps to reproduce + + + +## Your environment + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..669221a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,27 @@ +--- +name: ✨ Feature request +about: Suggest an idea to implement +labels: enhancement +--- + +## User Story + + + +## Example + + + + +## Definition of Done + diff --git a/.github/workflows/check-image.yml b/.github/workflows/check-image.yml new file mode 100644 index 0000000..d64dc01 --- /dev/null +++ b/.github/workflows/check-image.yml @@ -0,0 +1,22 @@ +name: Build and check image + +on: push + +jobs: + verify-image-build: + runs-on: ubuntu-latest + steps: + - name: Checkout repo content + uses: actions/checkout@v2 + - name: ooil version + uses: docker://itisfoundation/ci-service-integration-library:v1.0.3-dev-4 + with: + args: ooil --version + - name: Assemble docker-compose spec + uses: docker://itisfoundation/ci-service-integration-library:v1.0.3-dev-4 + with: + args: ooil compose + - name: Build all images if multiple + uses: docker://itisfoundation/ci-service-integration-library:v1.0.3-dev-4 + with: + args: docker-compose build diff --git a/.osparc/docker-compose.overwrite.yml b/.osparc/docker-compose.overwrite.yml new file mode 100644 index 0000000..a0c27b4 --- /dev/null +++ b/.osparc/docker-compose.overwrite.yml @@ -0,0 +1,5 @@ +version: "3.7" +services: + jupyter-axondeepseg: + build: + dockerfile: Dockerfile diff --git a/.osparc/metadata.yml b/.osparc/metadata.yml new file mode 100644 index 0000000..ca448a7 --- /dev/null +++ b/.osparc/metadata.yml @@ -0,0 +1,197 @@ +name: AxonDeepSeg in JupyterLab +key: simcore/services/dynamic/jupyter-axondeepseg +type: dynamic +integration-version: 2.0.0 +version: 1.0.0 +description: AxonDeepSeg in JupyterLab +contact: iavarone@itis.swiss +thumbnail: https://raw.githubusercontent.com/axondeepseg/doc-figures/main/index/fig2.png +authors: + - name: Elisabetta Iavarone + email: iavarone@itis.swiss + affiliation: IT'IS Foundation +inputs: + input_1: + displayOrder: 1 + label: input_file_1 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_2: + displayOrder: 2 + label: input_file_2 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_3: + displayOrder: 3 + label: input_file_3 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_4: + displayOrder: 4 + label: input_file_4 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_4: + displayOrder: 4.0 + label: input_files_4 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_5: + displayOrder: 5.0 + label: input_files_5 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_6: + displayOrder: 6.0 + label: input_files_6 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_7: + displayOrder: 7.0 + label: input_files_7 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_8: + displayOrder: 8.0 + label: input_files_8 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_9: + displayOrder: 9.0 + label: input_files_9 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_10: + displayOrder: 10.0 + label: input_files_10 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_11: + displayOrder: 11.0 + label: input_files_11 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_12: + displayOrder: 12.0 + label: input_files_12 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_13: + displayOrder: 13.0 + label: input_files_13 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_14: + displayOrder: 14.0 + label: input_files_14 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_15: + displayOrder: 15.0 + label: input_files_15 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_16: + displayOrder: 16.0 + label: input_files_16 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_17: + displayOrder: 17.0 + label: input_files_17 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_18: + displayOrder: 18.0 + label: input_files_18 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_19: + displayOrder: 19.0 + label: input_files_19 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* + input_20: + displayOrder: 20.0 + label: input_files_20 + description: + Any input files. One or several files compressed in a zip will be + downloaded in an inputs folder. + type: data:*/* +outputs: + output_1: + displayOrder: 1 + label: output_files_1 + description: Output files uploaded from the outputs folder + type: data:*/* + output_2: + displayOrder: 2 + label: output_files_2 + description: Output files uploaded from the outputs folder + type: data:*/* + output_3: + displayOrder: 3 + label: output_files_3 + description: Output files uploaded from the outputs folder + type: data:*/* + output_4: + displayOrder: 4 + label: output_files_4 + description: Output files uploaded from the outputs folder + type: data:*/* + +boot-options: + boot_mode: + label: Boot mode + description: Select boot type for the service + default: "0" + items: + "0": + label: JupyterLab + description: Display the JupyterLab interface the default boot mode + "1": + label: Voila + description: + To start as Voila save a notebook as "voila.ipynb" in the root + folder +min-visible-inputs: 4 \ No newline at end of file diff --git a/.osparc/runtime.yml b/.osparc/runtime.yml new file mode 100644 index 0000000..0975a1a --- /dev/null +++ b/.osparc/runtime.yml @@ -0,0 +1,24 @@ +restart-policy: no-restart +settings: + - name: ports + type: int + value: 8888 + - name: constraints + type: string + value: + - node.platform.os == linux + - name: Resources + type: Resources + value: + Limits: + NanoCPUs: 8000000000 + MemoryBytes: 32000000000 + GenericResources: + - DiscreteResourceSpec: + Kind: AIRAM + Value: 1 +paths-mapping: + inputs_path: /home/jovyan/work/inputs + outputs_path: /home/jovyan/work/outputs + state_paths: + - /home/jovyan/work/workspace diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ed1395a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## [1.0.0] - 2024-02 +- First version +- AxonDeepSeg commit 9ab16452a1411788e845e5c1e2cd2301ba5e81b5 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..93738c3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,241 @@ +# Starting from CUDA image, then install JupyterLab (as done in the official Jupyter images), and AxonDeepSeg (requires python3.8), torch with CUDA +FROM nvidia/cudagl:11.2.2-devel-ubuntu20.04 as base + +LABEL maintainer=elisabettai + +#------------------------ docker-stacks-foundation--------------------------------------- +# https://github.com/jupyter/docker-stacks/blob/main/images/docker-stacks-foundation/Dockerfile + +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for the Server that starts +# but lacks all features (e.g., download as all possible file formats) +ENV DEBIAN_FRONTEND noninteractive +RUN apt-key adv --fetch-keys 'https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub' &&\ + apt-get update --yes && \ + # - `apt-get upgrade` is run to patch known vulnerabilities in system packages + # as the Ubuntu base image is rebuilt too seldom sometimes (less than once a month) + apt-get upgrade --yes && \ + apt-get install --yes --no-install-recommends \ + # - bzip2 is necessary to extract the micromamba executable. + bzip2 \ + ca-certificates \ + locales \ + sudo \ + gosu \ + # - `tini` is installed as a helpful container entrypoint, + # that reaps zombie processes and such of the actual executable we want to start + # See https://github.com/krallin/tini#why-tini for details + #tini \ + wget && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen + +# Configure environment +ENV CONDA_DIR=/opt/conda \ + SHELL=/bin/bash \ + NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV PATH="${CONDA_DIR}/bin:${PATH}" \ + HOME="/home/${NB_USER}" + +# Copy a script that we will use to correct permissions after running certain commands +COPY original_jupyter_scripts/fix-permissions /usr/local/bin/fix-permissions +RUN chmod a+rx /usr/local/bin/fix-permissions + +# Enable prompt color in the skeleton .bashrc before creating the default NB_USER +# hadolint ignore=SC2016 +RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ + # More information in: https://github.com/jupyter/docker-stacks/pull/2047 + # and docs: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html + echo 'eval "$(conda shell.bash hook)"' >> /etc/skel/.bashrc + +# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group +# and make sure these dirs are writable by the `users` group. +RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ + sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ + sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ + useradd --no-log-init --create-home --shell /bin/bash --uid "${NB_UID}" --no-user-group "${NB_USER}" && \ + mkdir -p "${CONDA_DIR}" && \ + chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ + chmod g+w /etc/passwd && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +USER ${NB_UID} + +# Pin the Python version here, or set it to "default" +ARG PYTHON_VERSION=3.8 + +# Setup work directory for backward-compatibility +RUN mkdir "/home/${NB_USER}/work" && \ + fix-permissions "/home/${NB_USER}" + +# Download and install Micromamba, and initialize the Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Install Python, Mamba, and jupyter_core +# Cleanup temporary files and remove Micromamba +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +COPY --chown="${NB_UID}:${NB_GID}" original_jupyter_scripts/initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + # https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html#linux-and-macos + wget --progress=dot:giga -O - \ + "https://micro.mamba.pm/api/micromamba/linux-${arch}/latest" | tar -xvj bin/micromamba && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # Install the packages + ./bin/micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --yes \ + "${PYTHON_SPECIFIER}" \ + 'mamba' \ + 'jupyter_core' && \ + rm -rf /tmp/bin/ && \ + # Pin major.minor version of python + # https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#preventing-packages-from-updating-pinning + mamba list --full-name 'python' | tail -1 | tr -s ' ' | cut -d ' ' -f 1,2 | sed 's/\.[^.]*$/.*/' >> "${CONDA_DIR}/conda-meta/pinned" && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Switch back to jovyan to avoid accidental container runs as root +USER ${NB_UID} + +WORKDIR "${HOME}" + +# ------------------------------- Install base Jupyter ------------------------------- +# From: https://github.com/jupyter/docker-stacks/blob/main/images/base-notebook/Dockerfile + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for the Server that starts +# but lacks all features (e.g., download as all possible file formats) +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + # - Add necessary fonts for matplotlib/seaborn + # See https://github.com/jupyter/docker-stacks/pull/380 for details + fonts-liberation \ + npm \ + # - `pandoc` is used to convert notebooks to html files + # it's not present in the aarch64 Ubuntu image, so we install it here + # git needed to clone AxonDeepSeg repo + git \ + pandoc && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +USER ${NB_UID} + +# Install JupyterLab, Jupyter Notebook, voila +# Generate a Jupyter Server config +# Cleanup temporary files +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +WORKDIR /tmp +RUN mamba install --yes \ + 'jupyterlab' \ + 'notebook' \ + 'jupyterlab_latex' \ + 'voila' \ + 'jupyterlab-git' && \ + jupyter server --generate-config && \ + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +ENV JUPYTER_PORT=8888 +EXPOSE $JUPYTER_PORT + +# Configure container startup and make them execuble +CMD ["start-notebook.py"] +COPY original_jupyter_scripts/start-notebook.py original_jupyter_scripts/start-notebook.sh /usr/local/bin/ + +USER root +RUN chmod +x /usr/local/bin/start-notebook.sh && \ + chmod +x /usr/local/bin/start-notebook.py +USER ${NB_UID} + +# -------------------------- Install AxonDeepSeg ------------------------- + +ENV ADS_SHA=9ab16452a1411788e845e5c1e2cd2301ba5e81b5 + +# Create conda env for axondeepseg +RUN git clone https://github.com/neuropoly/axondeepseg.git && \ + cd axondeepseg && \ + git checkout -b release ${ADS_SHA} && \ + conda init bash && \ + conda env create + +# "Activate" the conda env +SHELL ["conda", "run", "-n", "ads_venv", "/bin/bash", "-c" ] + +RUN cd axondeepseg && \ + pip install -e . plugins/ && \ + axondeepseg_test + +# Create a jupyter kernel for AxonDeepSeg +RUN conda install -c conda-forge ipykernel && \ + python -m ipykernel install --user --name=ads_venv + +# Install torch for AxonDeepSeg+GPU +# Versions from https://pytorch.org/get-started/previous-versions/. No CUDA 11.2 mentioned, so using latest torch version for CUDA 11.3 +#RUN conda install pytorch==1.12.0 torchvision==0.13.0 torchaudio==0.12.0 cudatoolkit=11.3 -c pytorch +RUN pip install torch==1.12.0+cu113 torchvision==0.13.0+cu113 torchaudio==0.12.0 --extra-index-url https://download.pytorch.org/whl/cu113 + +# Activate AxonDeepSeg conda env at startup +RUN echo "source /opt/conda/bin/activate ads_venv" >> ~/.bashrc + + +# -------------------------- Final setup --------------------------------- +USER root + +ENV JUPYTER_ENABLE_LAB="yes" +# autentication is disabled for now +ENV NOTEBOOK_TOKEN="" +ENV NOTEBOOK_BASE_DIR="$HOME/work" + +# copy README and CHANGELOG +COPY --chown=$NB_UID:$NB_GID README.ipynb ${NOTEBOOK_BASE_DIR}/README.ipynb +COPY --chown=$NB_UID:$NB_GID CHANGELOG.md ${NOTEBOOK_BASE_DIR}/CHANGELOG.md + +# remove write permissions from files which are not supposed to be edited +RUN chmod gu-w ${NOTEBOOK_BASE_DIR}/CHANGELOG.md + +RUN mkdir --parents "/home/${NB_USER}/.virtual_documents" && \ + chown --recursive "$NB_USER" "/home/${NB_USER}/.virtual_documents" +ENV JP_LSP_VIRTUAL_DIR="/home/${NB_USER}/.virtual_documents" + +# Copying boot scripts +COPY --chown=$NB_UID:$NB_GID boot_scripts/ /docker + +ENTRYPOINT [ "/bin/bash", "/docker/entrypoint.bash" ] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..64e4200 --- /dev/null +++ b/Makefile @@ -0,0 +1,127 @@ +# Author: Elisabetta Iavarone + +SHELL = /bin/sh +.DEFAULT_GOAL := help + +export VCS_URL := $(shell git config --get remote.origin.url 2> /dev/null || echo unversioned repo) +export VCS_REF := $(shell git rev-parse --short HEAD 2> /dev/null || echo unversioned repo) +export VCS_STATUS := $(if $(shell git status -s 2> /dev/null || echo unversioned repo),'modified/untracked','clean') +export BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") + +export DOCKER_IMAGE_NAME ?= axondeepseg-in-jupyterlab +export DOCKER_IMAGE_TAG ?= 1.0.0 + +OSPARC_DIR:=$(CURDIR)/.osparc + +APP_NAME := jupyter-axondeepseg + + +# PYTHON ENVIRON --------------------------------------------------------------------------------------- +.PHONY: devenv +.venv: + @python3 --version + python3 -m venv $@ + # upgrading package managers + $@/bin/pip install --upgrade \ + pip \ + wheel \ + setuptools + +devenv: .venv ## create a python virtual environment with tools to dev, run and tests cookie-cutter + # installing extra tools + @$/dev/null || echo not found)' + @echo ' make : $(shell make --version 2>&1 | head -n 1)' + @echo ' jq : $(shell jq --version 2>/dev/null || echo not found z)' + @echo ' awk : $(shell awk -W version 2>&1 | head -n 1 2>/dev/null || echo not found)' + @echo ' python : $(shell python3 --version 2>/dev/null || echo not found )' + @echo ' docker : $(shell docker --version)' + @echo ' docker buildx : $(shell docker buildx version)' + @echo ' docker-compose : $(shell docker-compose --version)' + +# MISC ----------------------------------------------------------------- + +.PHONY: clean +git_clean_args = -dxf --exclude=.vscode/ + +clean: ## cleans all unversioned files in project and temp files create by this makefile + # Cleaning unversioned + @git clean -n $(git_clean_args) + @echo -n "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ] + @echo -n "$(shell whoami), are you REALLY sure? [y/N] " && read ans && [ $${ans:-N} = y ] + @git clean $(git_clean_args) diff --git a/README.ipynb b/README.ipynb new file mode 100644 index 0000000..8388079 --- /dev/null +++ b/README.ipynb @@ -0,0 +1,61 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "67bb6fe5-38a1-472e-bf7c-9a300da42cb7", + "metadata": {}, + "source": [ + "# Before starting\n", + "\n", + "#### Directories and usage:\n", + "* **You are here:** `~/work/workspace` this is the current directory when the JupyterLab is opened and where you should save your files. \n", + "\n", + "* **Inputs from upstream services:** `~/work/inputs` (equivalent to `/home/jovyan/work/inputs`) contains inputs incoming from the outputs of the previous node. *Warning: do not write files in this directory or they will be erased.* *Jovyan* is the username in this Jupyterlab (and it is the same for all o²S²PARC users). Note: you cannot write the the `inputs` folder.\n", + "\n", + "* **This service's Outputs:** the contents of `~/work/outputs/outputs_1`, `~/work/outputs/outputs_2`, `~/work/outputs/outputs_3` and `~/work/outputs/outputs_4` will be accessible to connected downstream nodes. *Warning: do not add files into `~/work/outputs` base directory or they will be erased.*\n", + "\n", + "#### Good Practices\n", + "* **Return to Dashboard often:** data are versioned ONLY when you have saved the file in JupyterLab AND after you return to the Dashboard from your study. We advise you to go back to the Dashboard often to avoid losing work.*\n", + "* **I don't see `inputs` or `outputs`:** `~/work/` (one folder up from the current directory) is where you can find the `inputs` and `outputs` directories. Write access to this folder has been disabled. You can only create files and directories in:\n", + " * `~/work/workspace`\n", + " * `~/work/outputs/outputs_1`\n", + " * `~/work/outputs/outputs_2`\n", + " * `~/work/outputs/outputs_3`\n", + " * `~/work/outputs/outputs_4`\n", + "* **Do not write files directly to outputs:** if your code generates files, we recommend writing them first to a temporary directory (e.g. in `~/work/workspace`) and then moving them to the desired `~/work/outputs/output_X` directory after completion. Otherwise, data synchronisation may start while the file is being written to, which may result in a corrupted file in output.\n", + "\n", + "#### Have an issue?\n", + "If you have a problem or a question, please open an issue on [Github](https://github.com/ITISFoundation/jupyterlab-AxonDeepSeg-osparc/issues). If you don't have a GitHub account, please write to support@osparc.io." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "fdc3059f", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python (maths)", + "language": "python", + "name": "python-maths" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..466f2a6 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# + +AxonDeepSeg in JupyterLab + +## Usage + +```console +$ make help + +$ make build +$ make run-local + +``` + +## Workflow +1. The [Dockerfile](jupyter-axondeepseg/src/Dockerfile) shall be modified to install additional packages and/or Jupyter kernels +2. The [.osparc](.osparc) is the configuration folder and source of truth for metadata: describes service info and expected inputs/outputs of the service. +3. The service docker image may be built with ``make build`` (see usage above) +4. The service docker image may be built with ``make run-local`` (see usage above) +5. If the image works correctly, you'll be able to access the JupyterLab interface in your browser at ``localhost:8888`` + +## Have an issue or question? +Please open an issue [in this repository](https://github.com/ITISFoundation/cookiecutter-osparc-service/issues/). +--- +

+Made with love at www.z43.swiss +

diff --git a/boot_scripts/boot_notebook.bash b/boot_scripts/boot_notebook.bash new file mode 100755 index 0000000..e8ef543 --- /dev/null +++ b/boot_scripts/boot_notebook.bash @@ -0,0 +1,87 @@ +#!/bin/bash +# SEE http://redsymbol.net/articles/unofficial-bash-strict-mode/ + +set -euo pipefail +IFS=$'\n\t' +INFO="INFO: [$(basename "$0")] " + +echo "$INFO" " User :$(id "$(whoami)")" +echo "$INFO" " Workdir :$(pwd)" + +# Trust all notebooks in the notebooks folder +echo "$INFO" "trust all notebooks in path..." +find "${NOTEBOOK_BASE_DIR}" -name '*.ipynb' -type f | xargs -I % /bin/bash -c 'jupyter trust "%" || true' || true + +# Configure +# Prevents notebook to open in separate tab +mkdir --parents "$HOME/.jupyter/custom" +cat > "$HOME/.jupyter/custom/custom.js" < .jupyter_config.json < "/opt/conda/share/jupyter/lab/overrides.json" < /dev/null 2>&1 || \ + (echo "ERROR: You must mount '${DY_SIDECAR_PATH_INPUTS}' to deduce user and group ids" && exit 1) +stat "${DY_SIDECAR_PATH_OUTPUTS}" > /dev/null 2>&1 || \ + (echo "ERROR: You must mount '${DY_SIDECAR_PATH_OUTPUTS}' to deduce user and group ids" && exit 1) + +# NOTE: expects docker run ... -v /path/to/input/folder:${DY_SIDECAR_PATH_INPUTS} +# check input/output folders are owned by the same user +if [ "$(stat -c %u "${DY_SIDECAR_PATH_INPUTS}")" -ne "$(stat -c %u "${DY_SIDECAR_PATH_OUTPUTS}")" ] +then + echo "ERROR: '${DY_SIDECAR_PATH_INPUTS}' and '${DY_SIDECAR_PATH_OUTPUTS}' have different user id's. not allowed" && exit 1 +fi +# check input/outputfolders are owned by the same group +if [ "$(stat -c %g "${DY_SIDECAR_PATH_INPUTS}")" -ne "$(stat -c %g "${DY_SIDECAR_PATH_OUTPUTS}")" ] +then + echo "ERROR: '${DY_SIDECAR_PATH_INPUTS}' and '${DY_SIDECAR_PATH_OUTPUTS}' have different group id's. not allowed" && exit 1 +fi + +echo "listing inputs folder ${DY_SIDECAR_PATH_INPUTS}" +ls -lah "${DY_SIDECAR_PATH_INPUTS}" +echo "listing outputs folder ${DY_SIDECAR_PATH_OUTPUTS}" +ls -lah "${DY_SIDECAR_PATH_OUTPUTS}" + +echo "setting correct user id/group id..." +HOST_USERID=$(stat -c %u "${DY_SIDECAR_PATH_INPUTS}") +HOST_GROUPID=$(stat -c %g "${DY_SIDECAR_PATH_INPUTS}") +CONTAINER_GROUPNAME=$(getent group | grep "${HOST_GROUPID}" | cut --delimiter=: --fields=1 || echo "") +echo "CONTAINER_GROUPNAME='$CONTAINER_GROUPNAME'" + +if [ "$HOST_USERID" -eq 0 ] +then + echo "Warning: Folder mounted owned by root user... adding $NB_USER to root..." + addgroup "$NB_USER" root +else + echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONTAINER_GROUPNAME'..." + # take host's credentials in $NB_USER + if [ -z "$CONTAINER_GROUPNAME" ] + then + echo "Creating new group my$NB_USER" + CONTAINER_GROUPNAME=my$NB_USER + addgroup --gid "$HOST_GROUPID" "$CONTAINER_GROUPNAME" + else + echo "group already exists" + fi + + echo "adding $NB_USER to group $CONTAINER_GROUPNAME..." + usermod --append --groups "$CONTAINER_GROUPNAME" "$NB_USER" + + echo "Chainging owner ship of state directory /home/${NB_USER}/work/workspace" + chown --recursive "$NB_USER" "/home/${NB_USER}/work/workspace" + echo "Chainging owner ship of state directory ${DY_SIDECAR_PATH_INPUTS}" + chown --recursive "$NB_USER" "${DY_SIDECAR_PATH_INPUTS}" + echo "Chainging owner ship of state directory ${DY_SIDECAR_PATH_OUTPUTS}" + chown --recursive "$NB_USER" "${DY_SIDECAR_PATH_OUTPUTS}" +fi + +mv "${NOTEBOOK_BASE_DIR}/README.ipynb" "${NOTEBOOK_BASE_DIR}/workspace/README.ipynb" || true + +echo "Removing write permissions from users in placed where they are not allowed to write:" +echo "- /home/${NB_USER}/work" +chmod gu-w "/home/${NB_USER}/work" + +echo +echo "$INFO" "Starting notebook ..." +exec gosu "$NB_USER" /docker/boot_notebook.bash diff --git a/docker-compose-local.yml b/docker-compose-local.yml new file mode 100644 index 0000000..68b9faa --- /dev/null +++ b/docker-compose-local.yml @@ -0,0 +1,16 @@ +version: '3.7' +services: + jupyter-axondeepseg: + image: simcore/services/dynamic/jupyter-axondeepseg:1.0.0 + ports: + - "8888:8888" + environment: + - DY_SIDECAR_PATH_INPUTS=/tmp/inputs + - DY_SIDECAR_PATH_OUTPUTS=/tmp/outputs + - DY_BOOT_OPTION_BOOT_MODE=0 + - DY_SIDECAR_PATH/home/jovyan/work/workspace + volumes: + - /tmp/.X11-unix:/tmp/.X11-unix + - ${PWD}/validation/workspace:/home/jovyan/work/workspace + - ${PWD}/validation/inputs:/tmp/inputs + - ${PWD}/validation/outputs:/tmp/outputs \ No newline at end of file diff --git a/original_jupyter_scripts/fix-permissions b/original_jupyter_scripts/fix-permissions new file mode 100644 index 0000000..d540462 --- /dev/null +++ b/original_jupyter_scripts/fix-permissions @@ -0,0 +1,33 @@ +#!/bin/bash +# Set permissions on a directory +# After any installation, if a directory needs to be (human) user-writable, run this script on it. +# It will make everything in the directory owned by the group ${NB_GID} and writable by that group. +# Deployments that want to set a specific user id can preserve permissions +# by adding the `--group-add users` line to `docker run`. + +# Uses find to avoid touching files that already have the right permissions, +# which would cause a massive image explosion + +# Right permissions are: +# group=${NB_GID} +# AND permissions include group rwX (directory-execute) +# AND directories have setuid,setgid bits set + +set -e + +for d in "$@"; do + find "${d}" \ + ! \( \ + -group "${NB_GID}" \ + -a -perm -g+rwX \ + \) \ + -exec chgrp "${NB_GID}" -- {} \+ \ + -exec chmod g+rwX -- {} \+ + # setuid, setgid *on directories only* + find "${d}" \ + \( \ + -type d \ + -a ! -perm -6000 \ + \) \ + -exec chmod +6000 -- {} \+ +done \ No newline at end of file diff --git a/original_jupyter_scripts/initial-condarc b/original_jupyter_scripts/initial-condarc new file mode 100644 index 0000000..66ecf60 --- /dev/null +++ b/original_jupyter_scripts/initial-condarc @@ -0,0 +1,6 @@ +# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge \ No newline at end of file diff --git a/original_jupyter_scripts/start-notebook.py b/original_jupyter_scripts/start-notebook.py new file mode 100644 index 0000000..be4e75e --- /dev/null +++ b/original_jupyter_scripts/start-notebook.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import shlex +import sys + +# If we are in a JupyterHub, we pass on to `start-singleuser.py` instead so it does the right thing +if "JUPYTERHUB_API_TOKEN" in os.environ: + print( + "WARNING: using start-singleuser.py instead of start-notebook.py to start a server associated with JupyterHub." + ) + command = ["/usr/local/bin/start-singleuser.py"] + sys.argv[1:] + os.execvp(command[0], command) + + +# Entrypoint is start.sh +command = [] + +# If we want to survive restarts, launch the command using `run-one-constantly` +if os.environ.get("RESTARTABLE") == "yes": + command.append("run-one-constantly") + +# We always launch a jupyter subcommand from this script +command.append("jupyter") + +# Launch the configured subcommand. +# Note that this should be a single string, so we don't split it. +# We default to `lab`. +jupyter_command = os.environ.get("DOCKER_STACKS_JUPYTER_CMD", "lab") +command.append(jupyter_command) + +# Append any optional NOTEBOOK_ARGS we were passed in. +# This is supposed to be multiple args passed on to the notebook command, +# so we split it correctly with shlex +if "NOTEBOOK_ARGS" in os.environ: + command += shlex.split(os.environ["NOTEBOOK_ARGS"]) + +# Pass through any other args we were passed on the command line +command += sys.argv[1:] + +# Execute the command! +print("Executing: " + " ".join(command)) +os.execvp(command[0], command) \ No newline at end of file diff --git a/original_jupyter_scripts/start-notebook.sh b/original_jupyter_scripts/start-notebook.sh new file mode 100644 index 0000000..6d4c1e6 --- /dev/null +++ b/original_jupyter_scripts/start-notebook.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Shim to emit warning and call start-notebook.py +echo "WARNING: Use start-notebook.py instead" + +exec /usr/local/bin/start-notebook.py "$@" \ No newline at end of file diff --git a/validation/inputs/.gitkeep b/validation/inputs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/validation/outputs/.gitkeep b/validation/outputs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/validation/workspace/.gitkeep b/validation/workspace/.gitkeep new file mode 100644 index 0000000..e69de29