From fc186534b951f5c363a6f5878e27c81974be62e3 Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 15:59:10 -0700 Subject: [PATCH 1/9] Update and rename apt.txt to apt2.txt --- apt.txt => apt2.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename apt.txt => apt2.txt (95%) diff --git a/apt.txt b/apt2.txt similarity index 95% rename from apt.txt rename to apt2.txt index 8290de3..7d3f3b1 100644 --- a/apt.txt +++ b/apt2.txt @@ -28,7 +28,8 @@ mc # Regular build tools for compiling common stuff # most installed with R installation -# need to install after R so these added to appendix +build-essentials +gfortran # Dependencies for nbconvert and myst # texlive is installed by R installation From 922a7a7ac3462f188d453365401113d43f751ad4 Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 16:01:29 -0700 Subject: [PATCH 2/9] Update apt2.txt --- apt2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apt2.txt b/apt2.txt index 7d3f3b1..d1d3329 100644 --- a/apt2.txt +++ b/apt2.txt @@ -28,7 +28,7 @@ mc # Regular build tools for compiling common stuff # most installed with R installation -build-essentials +build-essential gfortran # Dependencies for nbconvert and myst From d24a4fd41343bd2009486089c8683ef0d396e24c Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 16:02:37 -0700 Subject: [PATCH 3/9] Update appendix --- appendix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/appendix b/appendix index 1af7e7c..4178791 100644 --- a/appendix +++ b/appendix @@ -13,9 +13,11 @@ RUN PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \ chmod +x ${REPO_DIR}/rocker.sh && \ ${REPO_DIR}/rocker.sh -# Install any missing build packages. Most are installed when R installed but get any missing packages -RUN apt-get update && \ - apt-get install --yes --no-install-recommends build-essential gfortran && \ +# Install linux packages after R installation since the R install scripts get rid of packages +# The package_list part is reading the file and doing clean-up to just have the list of packages +RUN package_list=$(grep -v '^\s*#' ${REPO_DIR}/apt-extras.txt | grep -v '^\s*$' | sed 's/\r//g; s/#.*//; s/^[[:space:]]*//; s/[[:space:]]*$//' | awk '{$1=$1};1') && \ + apt-get update && \ + apt-get install --yes --no-install-recommends $package_list && \ apt-get autoremove --purge && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* From b583202ecfba2c77a5e46ae84ce0d8a6294090df Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 16:04:36 -0700 Subject: [PATCH 4/9] Update appendix --- appendix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appendix b/appendix index 4178791..0156fdb 100644 --- a/appendix +++ b/appendix @@ -15,7 +15,7 @@ RUN PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \ # Install linux packages after R installation since the R install scripts get rid of packages # The package_list part is reading the file and doing clean-up to just have the list of packages -RUN package_list=$(grep -v '^\s*#' ${REPO_DIR}/apt-extras.txt | grep -v '^\s*$' | sed 's/\r//g; s/#.*//; s/^[[:space:]]*//; s/[[:space:]]*$//' | awk '{$1=$1};1') && \ +RUN package_list=$(grep -v '^\s*#' ${REPO_DIR}/apt2.txt | grep -v '^\s*$' | sed 's/\r//g; s/#.*//; s/^[[:space:]]*//; s/[[:space:]]*$//' | awk '{$1=$1};1') && \ apt-get update && \ apt-get install --yes --no-install-recommends $package_list && \ apt-get autoremove --purge && \ From e88cb70ae74bf07f8b5f55b94618722de88a1c31 Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 16:13:01 -0700 Subject: [PATCH 5/9] Delete postBuild --- postBuild | 9 --------- 1 file changed, 9 deletions(-) delete mode 100755 postBuild diff --git a/postBuild b/postBuild deleted file mode 100755 index eea666a..0000000 --- a/postBuild +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -l -set -euo pipefail - -# Put our custom Jupyter Server config into appropriate platform -# This path is determined by looking at output of `jupyter --path` in the hub -# We copy both to notebook server config and jupyter server config, because either can be -# used in the JupyterHub. -cp custom_jupyter_server_config.json ${NB_PYTHON_PREFIX}/etc/jupyter/jupyter_server_config.d/ -cp custom_jupyter_server_config.json ${NB_PYTHON_PREFIX}/etc/jupyter/jupyter_notebook_config.d/ From 12c130ffc667ef82f0220d862a48008d7eee715e Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 16:42:03 -0700 Subject: [PATCH 6/9] Update appendix --- appendix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appendix b/appendix index 0156fdb..51287a6 100644 --- a/appendix +++ b/appendix @@ -2,6 +2,8 @@ USER root # repo2docker does not set this. This is the default env in repo2docker type images ENV CONDA_ENV=notebook +# Tell applications where to open desktop apps - this allows notebooks to pop open GUIs +ENV DISPLAY=":1.0" # Install R, RStudio via Rocker scripts ENV R_VERSION="4.4.1" From ddeb0fbe1c26a721ed15eafab2cecf4554229e2e Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 16:43:28 -0700 Subject: [PATCH 7/9] Update start --- start | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/start b/start index 48314f8..11ba4ee 100644 --- a/start +++ b/start @@ -4,13 +4,11 @@ set -euo pipefail # Start - Set any environment variables here # These are inherited by all processes, *except* RStudio # USE export =value -# Tell applications where to open desktop apps - this allows notebooks to pop open GUIs -export DISPLAY=":1.0" # source this file to get the variables defined in the rocker Dockerfile +# this is a series of export cmds source ${REPO_DIR}/env.txt # End - Set any environment variables here - # The for loops will fail if they return null (no files). Set shell option nullglob shopt -s nullglob From 6cfa515617bf3f263a2441a41f4361c25672d854 Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 19:39:28 -0700 Subject: [PATCH 8/9] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73bf193..1662922 100644 --- a/README.md +++ b/README.md @@ -71,4 +71,4 @@ USER ${NB_USER} The original [py-rocket 1.0](https://github.com/NASA-Openscapes/py-rocket) was developed by Luis Lopez and was built off a Rocker base image. Carl Boettiger and Eli Holmes later altered the image (py-rocket 2.0) so that the Python environment matched the Pangeo image structure but the image was still built off a Rocker image. Subsequently, Carl Boettiger developed [repo2docker-r](https://github.com/boettiger-lab/repo2docker-r) that creates a JupyterHub-compatible image that uses a [Jupyter docker stack image](https://jupyter-docker-stacks.readthedocs.io/en/latest/) as base. For py-rocker 3.0, Eli Holmes used Carl's ideas but used [repo2docker](https://repo2docker.readthedocs.io/en/latest/) and [repo2docker-action](https://github.com/jupyterhub/repo2docker-action) to build the base image. To do this, the [CryoCloud hub image](https://github.com/CryoInTheCloud/hub-image) repo was used for the basic structure and approach. Eli added the `rocker.sh` script and `appendix` modifications to install R and RStudio via the Rocker scripts (rather than using a Rocker image as base). Yuvi Panda (repo2docker) gave input throughout the process as snags were hit. -**Why Rocker for the R/RStudio environment?** The Rocker images are the standard for R/RStudio contanier images. They are heavily tested and regularly updated. There is a large developer community that fixes problems and bugs. The stack has gone through major revisions to improve modularity and they constantly innovating (integration for machine-learning, CUDA, BLAS, spatial, etc., etc.). py-rocker is building off that work without using the images directly. Instead it uses the Dockerfile code and the installation scripts. There are many other approaches to adding R and RStudio to images that work in JupyterHubs. See [repo2docker-r](https://github.com/boettiger-lab/repo2docker-r) that Carl developed and [r-conda](https://github.com/binder-examples/r-conda) for a conda native approach using repo2docker. py-rocket is not intended to create small images; it is intended to create images that emulate Rocker in the `/rstudio` environment on a JupyterHub. +**Why Rocker for the R/RStudio environment?** The Rocker images are the standard for R/RStudio contanier images. They are heavily tested and regularly updated. There is a large developer community that fixes problems and bugs. The stack has gone through major revisions to improve modularity and they constantly innovating (integration for machine-learning, CUDA, BLAS, spatial, etc., etc.). py-rocket is building off that work without using the images directly. Instead it uses the Docker file code and the installation scripts. There are many other approaches to adding R and RStudio to images that work in JupyterHubs. See [repo2docker-r](https://github.com/boettiger-lab/repo2docker-r) that Carl developed and [r-conda](https://github.com/binder-examples/r-conda) for a conda native approach using repo2docker. py-rocket is not intended to create small images; it is intended to create images that emulate Rocker in the `/rstudio` environment on a JupyterHub. From 23e491d81b6f19decd7c0008dc0d9b822decb87a Mon Sep 17 00:00:00 2001 From: Eli Holmes - NOAA Date: Tue, 29 Oct 2024 20:00:54 -0700 Subject: [PATCH 9/9] get rid of binder stuff just confusing --- appendix | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/appendix b/appendix index 51287a6..2db1013 100644 --- a/appendix +++ b/appendix @@ -48,29 +48,17 @@ ENV NB_USER=${NB_USER} ## ONBUILD section. These commands are run in child Dockerfiles. These are run right after the FROM image is loaded +ONBUILD RUN echo "py-rocket-base: looking for configuration files to use" + ONBUILD USER ${NB_USER} # ${REPO_DIR} is owned by ${NB_USER} ONBUILD COPY --chown=${NB_USER}:${NB_USER} . ${REPO_DIR}/childimage -# repo2docker will load files from a .binder or binder directory if -# present. We check if those directories exist. -ONBUILD RUN echo "Checking for 'binder' or '.binder' subfolder" \ - ; cd "${REPO_DIR}/childimage/" \ - ; if [ -d binder ] ; then \ - echo "Using 'binder/' build context" \ - ; elif [ -d .binder ] ; then \ - echo "Using '.binder/' build context" \ - ; else \ - echo "Using './' build context" \ - ; fi - # Copy Desktop files into ${REPO_DIR}/Desktop if they exist. start will copy to Application dir and Desktop # Will not fail if Desktop dir exists but is empty ONBUILD RUN echo "Checking for 'Desktop directory'..." \ ; cd "${REPO_DIR}/childimage/" \ - ; [ -d binder ] && cd binder \ - ; [ -d .binder ] && cd .binder \ ; if test -d Desktop ; then \ mkdir -p "${REPO_DIR}/Desktop" && \ [ "$(ls -A Desktop 2>/dev/null)" ] && cp -r Desktop/* "${REPO_DIR}/Desktop/"; \ @@ -81,8 +69,6 @@ ONBUILD RUN echo "Checking for 'Desktop directory'..." \ ONBUILD USER root ONBUILD RUN echo "Checking for 'apt.txt'..." \ ; cd "${REPO_DIR}/childimage/" \ - ; [ -d binder ] && cd binder \ - ; [ -d .binder ] && cd .binder \ ; if test -f "apt.txt" ; then \ package_list=$(grep -v '^\s*#' apt.txt | grep -v '^\s*$' | sed 's/\r//g; s/#.*//; s/^[[:space:]]*//; s/[[:space:]]*$//' | awk '{$1=$1};1') \ && apt-get update --fix-missing > /dev/null \ @@ -97,8 +83,6 @@ ONBUILD USER ${NB_USER} # sometimes package solving will get rid of pip installed packages. Make sure jupyter-remote-desktop-proxy does not disappear ONBUILD RUN echo "Checking for 'conda-lock.yml' or 'environment.yml'..." \ ; cd "${REPO_DIR}/childimage/" \ - ; [ -d binder ] && cd binder \ - ; [ -d .binder ] && cd .binder \ ; if test -f "conda-lock.yml" ; then echo "Using conda-lock.yml" & \ conda-lock install --name ${CONDA_ENV} \ && pip install --no-deps jupyter-remote-desktop-proxy \ @@ -118,8 +102,6 @@ ONBUILD RUN echo "Checking for 'conda-lock.yml' or 'environment.yml'..." \ # to avoid wasting space. ONBUILD RUN echo "Checking for pip 'requirements.txt'..." \ ; cd "${REPO_DIR}/childimage/" \ - ; [ -d binder ] && cd binder \ - ; [ -d .binder ] && cd .binder \ ; if test -f "requirements.txt" ; then \ ${NB_PYTHON_PREFIX}/bin/pip install --no-cache -r requirements.txt \ ; fi @@ -127,8 +109,6 @@ ONBUILD RUN echo "Checking for pip 'requirements.txt'..." \ # Add the r packages ONBUILD RUN echo "Checking for 'install.R" \ ; cd "${REPO_DIR}/childimage/" \ - ; [ -d binder ] && cd binder \ - ; [ -d .binder ] && cd .binder \ ; if test -f "install.R" ; then echo "Using install.R" & \ Rscript install.R \ ; fi @@ -137,8 +117,6 @@ ONBUILD RUN echo "Checking for 'install.R" \ # This is sourced in ${REPO_DIR}/start ONBUILD RUN echo "Checking for 'start'..." \ ; cd "${REPO_DIR}/childimage/" \ - ; [ -d binder ] && cd binder && cp start ${REPO_DIR}/childimage/start \ - ; [ -d .binder ] && cd .binder && cp start ${REPO_DIR}/childimage/start \ ; if test -f "start" ; then \ chmod +x start \ ; fi @@ -149,8 +127,6 @@ ONBUILD RUN echo "Checking for 'start'..." \ # leave behind. ONBUILD RUN echo "Checking for 'postBuild'..." \ ; cd "${REPO_DIR}/childimage/" \ - ; [ -d binder ] && cd binder \ - ; [ -d .binder ] && cd .binder \ ; if test -f "postBuild" ; then \ chmod +x postBuild \ && ./postBuild \