From def7c5bd0cc59dcc8c25ade4472f39c3d0c1506d Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 14 Mar 2022 14:21:11 -0400 Subject: [PATCH 1/5] Add GPU support to bioconductor docker --- bioc_scripts/add_bioc_devel_env_variables.sh | 9 ++++ bioc_scripts/install_bioc_sysdeps.sh | 3 +- .../bioc_devel.Dockerfile | 0 dockerfiles/bioc_ml_devel.Dockerfile | 49 +++++++++++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 bioc_scripts/add_bioc_devel_env_variables.sh rename Dockerfile => dockerfiles/bioc_devel.Dockerfile (100%) create mode 100644 dockerfiles/bioc_ml_devel.Dockerfile diff --git a/bioc_scripts/add_bioc_devel_env_variables.sh b/bioc_scripts/add_bioc_devel_env_variables.sh new file mode 100644 index 0000000..f591f03 --- /dev/null +++ b/bioc_scripts/add_bioc_devel_env_variables.sh @@ -0,0 +1,9 @@ +# Variables in Renviron.site are made available inside of R. +# Add libsbml CFLAGS + +curl -O http://bioconductor.org/checkResults/devel/bioc-LATEST/Renviron.bioc \ + && cat Renviron.bioc | grep -o '^[^#]*' | sed 's/export //g' >>/etc/environment \ + && cat Renviron.bioc >> /usr/local/lib/R/etc/Renviron.site \ + && echo 'LIBSBML_CFLAGS="-I/usr/include"' >> /usr/local/lib/R/etc/Renviron.site \ + && echo 'LIBSBML_LIBS="-lsbml"' >> /usr/local/lib/R/etc/Renviron.site \ + && rm -rf Renviron.bioc \ No newline at end of file diff --git a/bioc_scripts/install_bioc_sysdeps.sh b/bioc_scripts/install_bioc_sysdeps.sh index c04951c..21cc3d1 100644 --- a/bioc_scripts/install_bioc_sysdeps.sh +++ b/bioc_scripts/install_bioc_sysdeps.sh @@ -125,7 +125,8 @@ apt-get -y --no-install-recommends install \ ocl-icd-opencl-dev ## Python installations -pip3 install sklearn pandas pyyaml +pip3 install pandas pyyaml +# sklearn \ ## libgdal is needed for sf apt-get install -y --no-install-recommends \ diff --git a/Dockerfile b/dockerfiles/bioc_devel.Dockerfile similarity index 100% rename from Dockerfile rename to dockerfiles/bioc_devel.Dockerfile diff --git a/dockerfiles/bioc_ml_devel.Dockerfile b/dockerfiles/bioc_ml_devel.Dockerfile new file mode 100644 index 0000000..a994ae3 --- /dev/null +++ b/dockerfiles/bioc_ml_devel.Dockerfile @@ -0,0 +1,49 @@ +# The suggested name for this image is: bioconductor/bioconductor_docker:devel +FROM rocker/ml:devel + +## Set Dockerfile version number +ARG BIOCONDUCTOR_VERSION=3.15 + +##### IMPORTANT ######## +## The PATCH version number should be incremented each time +## there is a change in the Dockerfile. +ARG BIOCONDUCTOR_PATCH=0 + +ARG BIOCONDUCTOR_DOCKER_VERSION=${BIOCONDUCTOR_VERSION}.${BIOCONDUCTOR_PATCH} + +LABEL name="bioconductor/bioconductor_docker" \ + version=$BIOCONDUCTOR_DOCKER_VERSION \ + url="https://github.com/Bioconductor/bioconductor_docker" \ + vendor="Bioconductor Project" \ + maintainer="maintainer@bioconductor.org" \ + description="Bioconductor docker image with system dependencies to install all packages." \ + license="Artistic-2.0" + +## Add Bioconductor Requirements +ADD bioc_scripts /tmp/bioc_scripts +# ADD bioc_scripts/install_bioc_sysdeps.sh /tmp/ +# ADD bioc_scripts/install.R /tmp/ +# ADD bioc_scripts/add_bioc_devel_env_variables.sh /tmp/ + +## Add host-site-library +RUN echo "R_LIBS=/usr/local/lib/R/host-site-library:\${R_LIBS}" > /usr/local/lib/R/etc/Renviron.site \ + && echo "BIOCONDUCTOR_VERSION=${BIOCONDUCTOR_VERSION}" >> /usr/local/lib/R/etc/Renviron.site \ + && echo "BIOCONDUCTOR_DOCKER_VERSION=${BIOCONDUCTOR_DOCKER_VERSION}" >> /usr/local/lib/R/etc/Renviron.site + +# 1. Install Bioconductor system depdencies +RUN bash /tmp/bioc_scripts/install_bioc_sysdeps.sh + +# 2. Install specific version of BiocManager and libraries +# RUN R -f /tmp/bioc_scripts/install.R + +# 3. DEVEL: Add sys env variables to DEVEL image +RUN bash /tmp/bioc_scripts/add_bioc_devel_env_variables.sh + +# Add Env variables for specific packages +ENV LIBSBML_CFLAGS="-I/usr/include" +ENV LIBSBML_LIBS="-lsbml" +ENV BIOCONDUCTOR_DOCKER_VERSION=$BIOCONDUCTOR_DOCKER_VERSION +ENV BIOCONDUCTOR_VERSION=$BIOCONDUCTOR_VERSION + +# Init command for s6-overlay +CMD ["/init"] From 2a9a6d3bbe4044e66971aa6b9fbd46fd34a4966c Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 24 Mar 2022 11:44:16 -0400 Subject: [PATCH 2/5] Add updates to Dockerfile --- bioc_scripts/install_bioc_sysdeps.sh | 3 +-- dockerfiles/bioc_ml_devel.Dockerfile | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/bioc_scripts/install_bioc_sysdeps.sh b/bioc_scripts/install_bioc_sysdeps.sh index 21cc3d1..372b2ca 100644 --- a/bioc_scripts/install_bioc_sysdeps.sh +++ b/bioc_scripts/install_bioc_sysdeps.sh @@ -125,8 +125,7 @@ apt-get -y --no-install-recommends install \ ocl-icd-opencl-dev ## Python installations -pip3 install pandas pyyaml -# sklearn \ +pip3 install pandas pyyaml scikit-learn ## libgdal is needed for sf apt-get install -y --no-install-recommends \ diff --git a/dockerfiles/bioc_ml_devel.Dockerfile b/dockerfiles/bioc_ml_devel.Dockerfile index a994ae3..907f7ca 100644 --- a/dockerfiles/bioc_ml_devel.Dockerfile +++ b/dockerfiles/bioc_ml_devel.Dockerfile @@ -21,9 +21,6 @@ LABEL name="bioconductor/bioconductor_docker" \ ## Add Bioconductor Requirements ADD bioc_scripts /tmp/bioc_scripts -# ADD bioc_scripts/install_bioc_sysdeps.sh /tmp/ -# ADD bioc_scripts/install.R /tmp/ -# ADD bioc_scripts/add_bioc_devel_env_variables.sh /tmp/ ## Add host-site-library RUN echo "R_LIBS=/usr/local/lib/R/host-site-library:\${R_LIBS}" > /usr/local/lib/R/etc/Renviron.site \ From 9c6159d1e2dbc501c94ccf821618d0c1e0fec523 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 8 Jun 2022 11:34:07 -0400 Subject: [PATCH 3/5] Add conda install --- bioc_scripts/conda-tensorflow-env.yaml | 13 +++++++++++++ bioc_scripts/install-conda.sh | 10 ++++++++++ dockerfiles/bioc_ml_devel.Dockerfile | 3 +++ 3 files changed, 26 insertions(+) create mode 100644 bioc_scripts/conda-tensorflow-env.yaml create mode 100644 bioc_scripts/install-conda.sh diff --git a/bioc_scripts/conda-tensorflow-env.yaml b/bioc_scripts/conda-tensorflow-env.yaml new file mode 100644 index 0000000..1c03af6 --- /dev/null +++ b/bioc_scripts/conda-tensorflow-env.yaml @@ -0,0 +1,13 @@ +name: conda-tensorflow-env +channels: + - defaults + - conda-forge +dependencies: + - python=3.9 + - anaconda + - pip + - git + ## Scientific Computing + - numpy + - scipy + - tensorflow-gpu \ No newline at end of file diff --git a/bioc_scripts/install-conda.sh b/bioc_scripts/install-conda.sh new file mode 100644 index 0000000..c05e40b --- /dev/null +++ b/bioc_scripts/install-conda.sh @@ -0,0 +1,10 @@ +wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm -f Miniconda3-latest-Linux-x86_64.sh \ + && echo PATH="/root/miniconda3/bin":$PATH >> .bashrc \ + && exec bash \ + && conda --version + +conda env create -f /tmp/bioc_scripts/conda-tensorflow-env.yaml \ No newline at end of file diff --git a/dockerfiles/bioc_ml_devel.Dockerfile b/dockerfiles/bioc_ml_devel.Dockerfile index 907f7ca..4b942e0 100644 --- a/dockerfiles/bioc_ml_devel.Dockerfile +++ b/dockerfiles/bioc_ml_devel.Dockerfile @@ -36,6 +36,9 @@ RUN bash /tmp/bioc_scripts/install_bioc_sysdeps.sh # 3. DEVEL: Add sys env variables to DEVEL image RUN bash /tmp/bioc_scripts/add_bioc_devel_env_variables.sh +# 4. Install conda +RUN bash /tmp/bioc_scripts/install-conda.sh + # Add Env variables for specific packages ENV LIBSBML_CFLAGS="-I/usr/include" ENV LIBSBML_LIBS="-lsbml" From 8df2ff7614da93391d4b4c4db80eb6aa4f4ab837 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 9 Jun 2022 19:08:26 -0400 Subject: [PATCH 4/5] Install conda in the machine learning image --- bioc_scripts/install-conda.sh | 27 ++++++++++++------- ...kerfile => bioconductor_docker.Dockerfile} | 0 ....Dockerfile => bioconductor_ml.Dockerfile} | 5 ++-- 3 files changed, 21 insertions(+), 11 deletions(-) rename dockerfiles/{bioc_devel.Dockerfile => bioconductor_docker.Dockerfile} (100%) rename dockerfiles/{bioc_ml_devel.Dockerfile => bioconductor_ml.Dockerfile} (95%) diff --git a/bioc_scripts/install-conda.sh b/bioc_scripts/install-conda.sh index c05e40b..629cd3c 100644 --- a/bioc_scripts/install-conda.sh +++ b/bioc_scripts/install-conda.sh @@ -1,10 +1,19 @@ -wget \ - https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && mkdir /root/.conda \ - && bash Miniconda3-latest-Linux-x86_64.sh -b \ - && rm -f Miniconda3-latest-Linux-x86_64.sh \ - && echo PATH="/root/miniconda3/bin":$PATH >> .bashrc \ - && exec bash \ - && conda --version +wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ + && /bin/bash ~/miniconda.sh -b -p /opt/conda \ + && rm -f ~/miniconda.sh -conda env create -f /tmp/bioc_scripts/conda-tensorflow-env.yaml \ No newline at end of file +## https://docs.anaconda.com/anaconda/install/multi-user/ - +## Multi user conda installation - +## Install conda for both rstudio user and root - +## make sure the "env" is available to both users +groupadd condausers +chgrp -R condausers /opt/conda +chmod 770 -R /opt/conda + +## Add rstudio to condausers group +adduser rstudio condausers + +## Make sure conda works on "rstudio" user as well +cat <<"EOF">>/etc/bash.bashrc +PATH=/opt/conda/bin:$PATH +EOF diff --git a/dockerfiles/bioc_devel.Dockerfile b/dockerfiles/bioconductor_docker.Dockerfile similarity index 100% rename from dockerfiles/bioc_devel.Dockerfile rename to dockerfiles/bioconductor_docker.Dockerfile diff --git a/dockerfiles/bioc_ml_devel.Dockerfile b/dockerfiles/bioconductor_ml.Dockerfile similarity index 95% rename from dockerfiles/bioc_ml_devel.Dockerfile rename to dockerfiles/bioconductor_ml.Dockerfile index 4b942e0..7408e62 100644 --- a/dockerfiles/bioc_ml_devel.Dockerfile +++ b/dockerfiles/bioconductor_ml.Dockerfile @@ -1,8 +1,8 @@ # The suggested name for this image is: bioconductor/bioconductor_docker:devel -FROM rocker/ml:devel +FROM rocker/ml:4.2.0 ## Set Dockerfile version number -ARG BIOCONDUCTOR_VERSION=3.15 +ARG BIOCONDUCTOR_VERSION=3.16 ##### IMPORTANT ######## ## The PATCH version number should be incremented each time @@ -38,6 +38,7 @@ RUN bash /tmp/bioc_scripts/add_bioc_devel_env_variables.sh # 4. Install conda RUN bash /tmp/bioc_scripts/install-conda.sh +ENV PATH="/opt/conda/bin:${PATH}" # Add Env variables for specific packages ENV LIBSBML_CFLAGS="-I/usr/include" From bcba26203da80e993be2cc032c5b42bd9fba5f41 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 28 Jul 2022 10:32:40 -0700 Subject: [PATCH 5/5] PR for adding GPU support to bioconductor docker --- .github/workflows/weekly-build-bioc-ml.yaml | 60 ++++++++++++++++++++ .scripts/weekly_build_ml.sh | 63 +++++++++++++++++++++ dockerfiles/bioconductor_ml.Dockerfile | 7 ++- 3 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/weekly-build-bioc-ml.yaml create mode 100644 .scripts/weekly_build_ml.sh diff --git a/.github/workflows/weekly-build-bioc-ml.yaml b/.github/workflows/weekly-build-bioc-ml.yaml new file mode 100644 index 0000000..67c9d77 --- /dev/null +++ b/.github/workflows/weekly-build-bioc-ml.yaml @@ -0,0 +1,60 @@ +name: Bioc ML-GPU image weekly build + +on: + push: + branches: + - 'add_gpu_support' + +jobs: + build: + runs-on: ubuntu-latest + steps: + + - name: Checkout rocker/versioned2 repo + uses: actions/checkout@v3 + with: + repository: rocker-org/rocker-versioned2 + path: rocker-versioned2 + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Build and export + uses: docker/build-push-action@v2 + with: + context: . + file: dockerfiles/r-ver_devel.Dockerfile + tags: rocker/r-ver:devel + outputs: type=docker,dest=/tmp/rverdevel.tar + - + name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: r-ver + path: /tmp/rverdevel.tar + + # - + # name: Checkout + # uses: actions/checkout@v2 + + use: + runs-on: ubuntu-latest + needs: build + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Download artifact + uses: actions/download-artifact@v2 + with: + name: myimage + path: /tmp + - + name: Load image + run: | + docker load --input /tmp/myimage.tar + docker image ls -a + + diff --git a/.scripts/weekly_build_ml.sh b/.scripts/weekly_build_ml.sh new file mode 100644 index 0000000..a3378cb --- /dev/null +++ b/.scripts/weekly_build_ml.sh @@ -0,0 +1,63 @@ +set -e +echo "Build bioconductor_docker image" + +############################################### +## Step 0: Set up variables and clone +## Rocker repo +ROCKER_REPO=https://github.com/rocker-org/rocker-versioned2 + +## git clone rocker +git clone --depth 1 $ROCKER_REPO + +############################################### +## 1. docker build rocker r-ver: repo + +cd rocker-versioned2/ + +################################################################# +## NOTE: This needs to change when it comes time for RELEASE_3_12 +## +## The R_VERSION in the dockerfile for r-ver needs to be changed to +## 'ENV R_VERSION=patched' during this release cycle. +## +## This will be done with the help of "sed" + +#sed -r -i 's/ENV R_VERSION=4.0.2/ENV R_VERSION=patched/g' dockerfiles/r-ver_4.0.2.Dockerfile + +################################################################# + +docker build -t rocker/r-ver:devel -f dockerfiles/r-ver_devel.Dockerfile . + +############################################### +## 2. docker build rocker/cuda:devel + +echo "*** Building rocker/cuda:devel *** \n" + +docker build -t rocker/cuda:devel -f dockerfiles/cuda_devel.Dockerfile . + +############################################### +## 3. docker build nitesh1989/bioconductor_ml:devel + +cd $GITHUB_WORKSPACE; + +echo "*** Building nitesh1989/bioconductor_ml *** \n" + +## increment version number with sed +sed -r -i 's/(^ARG BIOCONDUCTOR_PATCH=)([0-9]+)$/echo "\1$((\2+1))"/ge' Dockerfile + +## Git login +git config user.email "bioc-issue-bot@bioconductor.org" +git config user.name "bioc-docker-bot" + +## Git commit +git commit -am "Weekly version bump and rebuild of nitesh1989/bioconductor_ml:devel" + +## docker build, login and push +docker build -t nitesh1989/bioconductor_ml:devel -f dockerfiles/bioc_ml_devel.Dockerfile . + +docker login --username=$DOCKER_USERNAME --password=$DOCKER_PASSWORD + +docker push nitesh1989/bioconductor_ml:devel + +## Finish +echo "Done" diff --git a/dockerfiles/bioconductor_ml.Dockerfile b/dockerfiles/bioconductor_ml.Dockerfile index 7408e62..cab070b 100644 --- a/dockerfiles/bioconductor_ml.Dockerfile +++ b/dockerfiles/bioconductor_ml.Dockerfile @@ -8,7 +8,6 @@ ARG BIOCONDUCTOR_VERSION=3.16 ## The PATCH version number should be incremented each time ## there is a change in the Dockerfile. ARG BIOCONDUCTOR_PATCH=0 - ARG BIOCONDUCTOR_DOCKER_VERSION=${BIOCONDUCTOR_VERSION}.${BIOCONDUCTOR_PATCH} LABEL name="bioconductor/bioconductor_docker" \ @@ -23,15 +22,17 @@ LABEL name="bioconductor/bioconductor_docker" \ ADD bioc_scripts /tmp/bioc_scripts ## Add host-site-library +## Set PIP_USER=false RUN echo "R_LIBS=/usr/local/lib/R/host-site-library:\${R_LIBS}" > /usr/local/lib/R/etc/Renviron.site \ && echo "BIOCONDUCTOR_VERSION=${BIOCONDUCTOR_VERSION}" >> /usr/local/lib/R/etc/Renviron.site \ - && echo "BIOCONDUCTOR_DOCKER_VERSION=${BIOCONDUCTOR_DOCKER_VERSION}" >> /usr/local/lib/R/etc/Renviron.site + && echo "BIOCONDUCTOR_DOCKER_VERSION=${BIOCONDUCTOR_DOCKER_VERSION}" >> /usr/local/lib/R/etc/Renviron.site \ + && echo "PIP_USER=false" >> /usr/local/lib/R/etc/Renviron.site # 1. Install Bioconductor system depdencies RUN bash /tmp/bioc_scripts/install_bioc_sysdeps.sh # 2. Install specific version of BiocManager and libraries -# RUN R -f /tmp/bioc_scripts/install.R +RUN R -f /tmp/bioc_scripts/install.R # 3. DEVEL: Add sys env variables to DEVEL image RUN bash /tmp/bioc_scripts/add_bioc_devel_env_variables.sh