simplify ENV management for both python and R (#3)

* Update devcontainer.json

* add new env

* fix path

* fix permission issue

* add readme

* fix container missing parts

* change to post-attach command to make sure it will work after restart

* update script

* fix minor issue

* change to relative path

.devcontainer/devcontainer.json

@@ -1,38 +1,38 @@
 {
     "name": "Python and R Dev Container",
-    "image": "mewu/data_access:latest",   // Use pre-built image
-//    "build": {
-//        "dockerfile": "../Dockerfile",
-//        "context": ".."
-//    },
+    "image": "mcr.microsoft.com/devcontainers/base:ubuntu",
+    "features": {
+        "ghcr.io/rocker-org/devcontainer-features/rstudio-server": {},
+        "ghcr.io/rocker-org/devcontainer-features/r-apt:latest": {},
+        "ghcr.io/rocker-org/devcontainer-features/r-packages:1": {
+        "packages": "knitr, markdown, tinytex"
+        },
+        "ghcr.io/devcontainers/features/python:1": {
+            "version": "3.12",
+            "installJupyterlab": true
+        },
+        "ghcr.io/rocker-org/devcontainer-features/quarto-cli:1": {},
+        "ghcr.io/rocker-org/devcontainer-features/pandoc:1": {},
+        "ghcr.io/rocker-org/devcontainer-features/renv-cache:0": {}
+    },
     "customizations": {
         "vscode": {
             "extensions": [
                 "ms-python.python",
                 "reditorsupport.r",
                 "quarto.quarto",
                 "ms-toolsai.jupyter"
-            ],
-            "settings": {
-                "python.defaultInterpreterPath": "/opt/venv/bin/python"
-            }
+            ]
         }
     },
-//    "workspaceFolder": "${localWorkspaceFolder}",
-//    "runArgs": [
-//     "--workdir", "/workspaces/${localWorkspaceFolder}"
-//     ],
-//    "mounts": [
-//    "source=${localWorkspaceFolder},target=/workspaces/${localWorkspaceFolder},type=bind,consistency=cached"
-//    ],
     "postCreateCommand": "bash ./.devcontainer/postCreateCommand.sh",
-	"postStartCommand": "bash ./.devcontainer/postStartCommand.sh",
+    "postAttachCommand": "bash ./.devcontainer/postAttachCommand.sh",
     "portsAttributes": {
         "8888": {
             "label": "Jupyter Lab"
         },
         "8787": {
-          "label": "R studio"
+          "label": "R Studio"
         }
     },
     "forwardPorts": [8787, 8888]

.devcontainer/postAttachCommand.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+echo "Restarting Rstudio server..."
+sudo rm -rf  /var/run/rstudio-server.id
+sleep 1
+rstudio-server start > rstudio_server.log 2>&1 &
+
+echo "Install quarto tinytex extension"
+quarto install tinytex
+
+pip install jupyterlab jupyter_core
+sleep 5
+echo "Setup Jupyter-lab"
+# Start Jupyter Lab
+if ! pgrep -x "jupyter-lab" > /dev/null; then
+    echo "Jupyter Lab is not started. Restarting..."
+    jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.token='' > jupyter_lab.log 2>&1 &
+fi
+
+echo "Dev environment startup completed!"

.devcontainer/postCreateCommand.sh

@@ -1,22 +1,23 @@
 #!/bin/bash
 
-# Activate the Python virtual environment
-source /opt/venv/bin/activate
+# Install ZeroMQ
+sudo apt-get update -y
+sudo apt-get install libzmq3-dev -y
 
-# Install Python dependencies
-if [ -f "python/requirements.txt" ]; then
-    pip install -r python/requirements.txt
-fi
+# Install GDAL
+sudo apt-get install gdal-bin libgdal-dev -y
 
-# Install R dependencies
-#if [ -f "r/install_packages.R" ]; then
-#    Rscript r/install_packages.R
-#fi
+# Install a few common dependencies
+sudo apt-get install libharfbuzz-dev libfribidi-dev libfreetype6-dev pkg-config libarchive-dev -y
+# Install ffmpeg
+sudo apt-get install ffmpeg -y
 
-# This may be done by the user
-# Set up renv for R
-#if [ -f "r/renv.lock" ]; then
-#    cd r && R -e 'renv::restore()'
-#fi
+# Setup bash to allow conda to run
+#conda init bash && conda install -y ipykernel
 
-echo "Dev environment setup completed!"
+# Allow JupyterLab and IRkernel ro run R workbook
+R -e "install.packages('IRkernel')"
+R -e "IRkernel::installspec()"
+
+# change the default working directory for RStudio
+echo "session-default-working-dir=/workspaces/data_access_library/" | sudo tee -a /etc/rstudio/rsession.conf

.github/workflows/publish.yml

@@ -28,54 +28,7 @@ jobs:
 
     - name: Run Quarto in Docker container
       run: |
-        docker run --rm -v ${{ github.workspace }}:/workspace -w /workspace mewu/data_access:latest bash -c "
-        
-          source /opt/venv/bin/activate
-          echo 'activated virtualenv.'
-          
-          chmod +x /workspace/r/install_quarto_extensions.expect
-          /workspace/r/install_quarto_extensions.expect
-          quarto list extensions
-          echo 'Check Quarto status:'
-          quarto check
-          echo 'Check R script path:'
-          which Rscript
-        
-          FOLDERS=(python r)
-          echo 'Folders to process: \${FOLDERS[@]}'
-
-          for FOLDER in \"\${FOLDERS[@]}\"; do
-            echo \"Searching for .qmd and .ipynb files in /workspace/\$FOLDER...\"
-            find \"/workspace/\$FOLDER\" \( -name '*.qmd' -o -name '*.ipynb' \) | while read -r file; do
-              
-              echo \"Processing file: \$file\"
-              # Remove the /workspace prefix from the file path
-              relative_dir=\$(dirname \"\${file#'/workspace/'}\")
-        
-              # Install extension to the folder directly
-              pushd \$relative_dir
-              /workspace/r/install_quarto_extensions.expect
-              popd
-        
-              target_dir=\"/workspace/gallery/\$relative_dir\"
-              echo \"Creating target directory: \$target_dir\"
-              mkdir -p \"\$target_dir\" && echo \"Directory \$target_dir created.\"
-
-              echo \"Attempting to render \$file with execution...\"
-              if quarto render \"\$file\" --to html --output-dir \"\$target_dir\"; then
-                echo \"Successfully rendered \$file with execution.\"
-              else
-                echo \"Failed to render \$file with execution. Rendering without execution...\"
-                if quarto render \"\$file\" --to html --no-execute --output-dir \"\$target_dir\"; then
-                  echo \"Successfully rendered \$file without execution.\"
-                else
-                  echo \"Rendering failed for \$file, even without execution.\"
-                fi
-              fi
-
-            done
-          done"
-
+        docker run --rm -v ${{ github.workspace }}:/workspace -w /workspace mewu/data_access:latest bash -c "chmod +x ./run_quarto_publish.sh && ./run_quarto_publish.sh"
 
     - name: Generate Index HTML
       run: |
@@ -101,6 +54,7 @@ jobs:
         git config --global user.email 'github-actions[bot]@users.noreply.github.com'
 
     - name: Deploy to GitHub Pages
+      if: github.ref == 'refs/heads/main'
       uses: peaceiris/actions-gh-pages@v4
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -25,6 +25,15 @@ RUN apt-get update && apt-get install -y \
     libharfbuzz-dev \
     libfribidi-dev \
     libfreetype6-dev \
+    libgl1-mesa-glx \
+    libegl1-mesa \
+    libxrandr2 \
+    libxss1 \
+    libxcursor1 \
+    libxcomposite1 \
+    libasound2 \
+    libxi6 \
+    libxtst6 \
     gdebi-core \
     software-properties-common \
     ffmpeg \
@@ -56,24 +65,39 @@ RUN /opt/venv/bin/python -m ipykernel install --user --name=venv --display-name
 RUN wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.57/quarto-1.5.57-linux-amd64.deb
 RUN dpkg -i quarto-1.5.57-linux-amd64.deb
 
-# Expose ports for RStudio (8787) and Jupyter (8888)
+# Install Anaconda
+# Set environment variable for non-interactive Anaconda install
+ENV PATH /opt/conda/bin:$PATH
+
+# Download and install Anaconda
+RUN wget --quiet https://repo.anaconda.com/archive/Anaconda3-2023.09-0-Linux-x86_64.sh -O ~/anaconda.sh && \
+    /bin/bash ~/anaconda.sh -b -p /opt/conda && \
+    rm ~/anaconda.sh && \
+    /opt/conda/bin/conda clean -i -t -p -y
+
+# Update conda to the latest version
+RUN conda update -n base -c defaults conda
+
+# Set the default shell to bash and activate Conda environment
+SHELL ["/bin/bash", "-c"]
+
+# Ensure conda is activated on startup
+RUN echo "source /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc
 
 WORKDIR /app
 
 COPY r /app/r
 COPY python /app/python
 
-# Install python packages in venv
-RUN /opt/venv/bin/pip install -r python/requirements.txt
-
+# Expose ports for RStudio (8787) and Jupyter (8888)
 EXPOSE 8787
 EXPOSE 8888
 
-# Install renv globally for all users
+# Install essentials globally for all users
 RUN R -e "install.packages('renv', repos='https://cran.rstudio.com/')"
-
-# Install R packages
-RUN Rscript /app/r/install_packages.R
+RUN R -e "install.packages('knitr', repos='https://cran.rstudio.com/')"
+RUN R -e "install.packages('rmarkdown', repos='https://cran.rstudio.com/')"
+RUN R -e ".libPaths()"
 
 # Add /usr/local/bin to the PATH for all users
 ENV PATH="/usr/local/bin:$PATH"