From abc7317c7db92291cb3d949fe0f06a4ec27843dc Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 21 Feb 2023 15:02:50 -0800 Subject: [PATCH 01/86] Fork the Workbench image for Google Cloud Workstations --- .../Dockerfile | 160 ++++++++++ .../Justfile | 86 ++++++ .../NEWS.md | 58 ++++ .../README.md | 279 ++++++++++++++++++ .../conf/jupyter.conf | 6 + .../conf/launcher-env | 6 + .../conf/launcher.conf | 12 + .../conf/launcher.local.conf | 1 + .../conf/launcher.pem | 28 ++ .../conf/launcher.pub | 9 + .../conf/logging.conf | 3 + .../conf/repos.conf | 2 + .../conf/rserver-float.conf | 1 + .../conf/rserver.conf | 13 + .../conf/vscode-user-settings.json | 5 + .../conf/vscode.conf | 4 + .../conf/vscode.extensions.conf | 3 + .../docker-compose.test.yml | 20 ++ .../pam/rstudio | 14 + .../pam/rstudio-session | 26 ++ .../sssd.conf | 9 + .../startup-launcher/rstudio-launcher.conf | 8 + .../startup-user-provisioning/sssd.conf | 11 + .../startup.sh | 78 +++++ .../startup/rstudio-workbench.conf | 8 + .../supervisord.conf | 47 +++ .../test/goss.yaml | 133 +++++++++ .../test/run_tests.sh | 32 ++ 28 files changed, 1062 insertions(+) create mode 100644 workbench-for-google-cloud-workstations/Dockerfile create mode 100644 workbench-for-google-cloud-workstations/Justfile create mode 100644 workbench-for-google-cloud-workstations/NEWS.md create mode 100644 workbench-for-google-cloud-workstations/README.md create mode 100644 workbench-for-google-cloud-workstations/conf/jupyter.conf create mode 100644 workbench-for-google-cloud-workstations/conf/launcher-env create mode 100644 workbench-for-google-cloud-workstations/conf/launcher.conf create mode 100644 workbench-for-google-cloud-workstations/conf/launcher.local.conf create mode 100644 workbench-for-google-cloud-workstations/conf/launcher.pem create mode 100644 workbench-for-google-cloud-workstations/conf/launcher.pub create mode 100644 workbench-for-google-cloud-workstations/conf/logging.conf create mode 100644 workbench-for-google-cloud-workstations/conf/repos.conf create mode 100644 workbench-for-google-cloud-workstations/conf/rserver-float.conf create mode 100644 workbench-for-google-cloud-workstations/conf/rserver.conf create mode 100644 workbench-for-google-cloud-workstations/conf/vscode-user-settings.json create mode 100644 workbench-for-google-cloud-workstations/conf/vscode.conf create mode 100644 workbench-for-google-cloud-workstations/conf/vscode.extensions.conf create mode 100644 workbench-for-google-cloud-workstations/docker-compose.test.yml create mode 100644 workbench-for-google-cloud-workstations/pam/rstudio create mode 100644 workbench-for-google-cloud-workstations/pam/rstudio-session create mode 100644 workbench-for-google-cloud-workstations/sssd.conf create mode 100644 workbench-for-google-cloud-workstations/startup-launcher/rstudio-launcher.conf create mode 100644 workbench-for-google-cloud-workstations/startup-user-provisioning/sssd.conf create mode 100644 workbench-for-google-cloud-workstations/startup.sh create mode 100644 workbench-for-google-cloud-workstations/startup/rstudio-workbench.conf create mode 100644 workbench-for-google-cloud-workstations/supervisord.conf create mode 100644 workbench-for-google-cloud-workstations/test/goss.yaml create mode 100755 workbench-for-google-cloud-workstations/test/run_tests.sh diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile new file mode 100644 index 00000000..84cc4296 --- /dev/null +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -0,0 +1,160 @@ +FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest + +### ARG declarations ### +ARG DEBIAN_FRONTEND=noninteractive +ARG R_VERSION=4.2.0 +ARG R_VERSION_ALT=3.6.2 +ARG PYTHON_VERSION=3.9.5 +ARG PYTHON_VERSION_ALT=3.8.10 +ARG PYTHON_VERSION_JUPYTER=3.8.10 +ARG TINI_VERSION=0.19.0 +ARG QUARTO_VERSION=1.1.251 +ARG DRIVERS_VERSION=2022.11.0 +ARG RSW_VERSION=2022.12.0+353.pro20 +ARG RSW_NAME=rstudio-workbench +ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/debian-11/amd64 + +ENV RSW_LICENSE "" +ENV RSW_LICENSE_SERVER "" +ENV RSW_TESTUSER rstudio +ENV RSW_TESTUSER_PASSWD rstudio +ENV RSW_TESTUSER_UID 10000 +ENV RSW_LAUNCHER true +ENV RSW_LAUNCHER_TIMEOUT 10 +ENV DIAGNOSTIC_DIR /var/log/rstudio +ENV DIAGNOSTIC_ENABLE false +ENV DIAGNOSTIC_ONLY false + +ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter + +### Update/upgrade system packages ### +RUN apt-get update --fix-missing \ + && apt-get upgrade -yq \ + && apt-get install -yq --no-install-recommends \ + dpkg-sig \ + krb5-user \ + libcap2 \ + libcurl4-gnutls-dev \ + libglib2.0-0 \ + libnss-sss \ + libpam-sss \ + libpq-dev \ + libpq5 \ + libsm6 \ + libuser \ + libuser1-dev \ + libxext6 \ + libxrender1 \ + locales \ + oddjob-mkhomedir \ + rrdtool \ + sssd \ + supervisor \ + && rm -rf /var/lib/apt/lists/* + +### Install tini ### +ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64 /tini +ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64.asc /tini.asc +RUN gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \ + && gpg --batch --verify /tini.asc /tini \ + && chmod +x /tini \ + && ln -s /tini /usr/local/bin/tini + +### Install R versions ### +RUN curl -O https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION}_1_amd64.deb \ + && curl -O https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ + && apt-get update \ + && apt-get install -yq --no-install-recommends ./r-${R_VERSION}_1_amd64.deb \ + && apt-get install -yq --no-install-recommends ./r-${R_VERSION_ALT}_1_amd64.deb \ + && rm -f ./r-${R_VERSION}_1_amd64.deb \ + && rm -f ./r-${R_VERSION_ALT}_1_amd64.deb \ + && ln -s /opt/R/${R_VERSION}/bin/R /usr/local/bin/R \ + && ln -s /opt/R/${R_VERSION}/bin/Rscript /usr/local/bin/Rscript + +### Install Python versions ### +RUN curl -O https://cdn.rstudio.com/python/debian-11/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ + && curl -O https://cdn.rstudio.com/python/debian-11/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \ + && apt-get update \ + && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION}_1_amd64.deb \ + && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \ + && rm -rf python-${PYTHON_VERSION}_1_amd64.deb \ + && rm -rf python-${PYTHON_VERSION_ALT}_1_amd64.deb \ + && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \ + && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools + +### Locale configuration ### +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +### Install Pro Drivers ### +RUN apt-get update \ + && apt-get install -yq --no-install-recommends unixodbc unixodbc-dev \ + && curl -O https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ + && apt-get update \ + && apt-get install -yq --no-install-recommends ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ + && rm -f ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ + && rm -rf /var/lib/apt/lists/* \ + && cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \ + && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/debian-11/latest")' + +### Install Workbench ### +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION//+/-}-amd64.deb" \ + && gpg --keyserver keyserver.ubuntu.com --recv-keys 3F32EE77E331692F \ + && dpkg-sig --verify ./rstudio-workbench.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends ./rstudio-workbench.deb \ + && rm ./rstudio-workbench.deb \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/rstudio-server/r-versions + +COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh +COPY startup-launcher/* /startup/launcher/ +COPY startup-user-provisioning/* /startup/user-provisioning/ +COPY startup/* /startup/base/ +COPY supervisord.conf /etc/supervisor/supervisord.conf + +RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyter \ + && /opt/python/jupyter/bin/pip install \ + jupyter \ + jupyterlab \ + rsconnect_jupyter \ + rsconnect_python \ + rsp_jupyter \ + workbench_jupyterlab \ + && ln -s /opt/python/jupyter/bin/jupyter /usr/local/bin/jupyter \ + && /opt/python/jupyter/bin/jupyter-nbextension install --sys-prefix --py rsp_jupyter \ + && /opt/python/jupyter/bin/jupyter-nbextension enable --sys-prefix --py rsp_jupyter \ + && /opt/python/jupyter/bin/jupyter-nbextension install --sys-prefix --py rsconnect_jupyter \ + && /opt/python/jupyter/bin/jupyter-nbextension enable --sys-prefix --py rsconnect_jupyter \ + && /opt/python/jupyter/bin/jupyter-serverextension enable --sys-prefix --py rsconnect_jupyter + +RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ + && chmod +x /usr/local/bin/wait-for-it.sh + +COPY --chmod=600 sssd.conf /etc/sssd/sssd.conf +COPY conf/* /etc/rstudio/ + +RUN mkdir -p /var/lib/rstudio-server/monitor/log \ + && chown -R rstudio-server:rstudio-server /var/lib/rstudio-server/monitor \ + && mkdir -p /startup/custom/ \ + && printf '\n# allow home directory creation\nsession required pam_mkhomedir.so skel=/etc/skel umask=0022' >> /etc/pam.d/common-session + +### Clean up ### +RUN apt-get remove -yq dpkg-sig \ + && apt-get install -yqf --no-install-recommends \ + && apt-get autoremove -yq \ + && apt-get clean -yq \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 80/tcp +EXPOSE 5559/tcp + +ENTRYPOINT [] +CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile new file mode 100644 index 00000000..9644785c --- /dev/null +++ b/workbench-for-google-cloud-workstations/Justfile @@ -0,0 +1,86 @@ +set positional-arguments + +BUILDX_PATH := "" + +IMAGE_PREFIX := "rstudio-" +PRODUCT := "workbench" +IMAGE_OS := "ubuntu1804" + +RSW_VERSION := "2022.12.0+353.pro20" +RSW_LICENSE := "" +RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" +RSW_LICENSE_SERVER := "" + +R_VERSION := "3.6.2" +R_VERSION_ALT := "4.1.0" + +PYTHON_VERSION := "3.9.5" +PYTHON_VERSION_ALT := "3.8.10" + +_make-default-tag: + echo "{{IMAGE_PREFIX}}{{PRODUCT}}:$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" + +# Build Workbench image - just build 2022.07.2+576.pro12 rstudio/rstudio-workbench-gcw:2022.07.2 +build VERSION=RSW_VERSION *TAGS="": + #!/usr/bin/env bash + set -euxo pipefail + BUILDX_ARGS="" + if [[ "{{BUILDX_PATH}}" != "" ]]; then + BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" + fi + + if [[ "{{TAGS}}" == "" ]]; then + raw_tag_array=($(just _make-default-tag)) + else + raw_tag_array=("{{TAGS}}") + fi + + tag_array=() + for tag in $raw_tag_array + do + tag_array+=("-t" $tag) + done + + docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ + ${tag_array[@]} \ + --build-arg RSW_VERSION="{{ VERSION }}" \ + --build-arg R_VERSION="{{ R_VERSION }}" \ + --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ + --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ + --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ + --build-arg RSW_DOWNLOAD_URL="{{ RSW_DOWNLOAD_URL }}" \ + --file=./Dockerfile . + +# Test Workbench image - just test rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 +test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": + #!/usr/bin/env bash + set -euxo pipefail + IMAGE_NAME="{{ TAG }}" \ + RSW_VERSION="{{ VERSION }}" \ + RSW_LICENSE="{{ RSW_LICENSE }}" \ + RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ + R_VERSION="{{ R_VERSION }}" \ + R_VERSION_ALT="{{ R_VERSION_ALT }}" \ + PYTHON_VERSION="{{ PYTHON_VERSION }}" \ + PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ + docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} + +# Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 +test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: + just test {{ TAG }} {{ VERSION }} bash + +# Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu1804-2022.07.2-576.pro12 +run TAG=`just _make-default-tag` CMD="": + #!/usr/bin/env bash + set -euxo pipefail + if [ -z "{{ RSW_LICENSE }}" ] && [ -z "{{ RSW_LICENSE_SERVER }}" ]; then + echo "Please set RSW_LICENSE or RSW_LICENSE_SERVER before running." + exit 1 + fi + + docker run -it --privileged \ + ${volume_opts[@]} \ + -p 80:8787 \ + -e RSW_LICENSE="{{ RSW_LICENSE }}" \ + -e RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ + "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-google-cloud-workstations/NEWS.md b/workbench-for-google-cloud-workstations/NEWS.md new file mode 100644 index 00000000..4afefe75 --- /dev/null +++ b/workbench-for-google-cloud-workstations/NEWS.md @@ -0,0 +1,58 @@ +# 2022-01-23 + +- Add documentation for license leak bug and possible workarounds/solutions. +- Add an option in the `Justfile` to persist license state files for use +across multiple restarts. + +# 2022.12.0 + +- Upgrade workbench to 2022.12.0 +- Use the new `WORKBENCH_JUPYTER_PATH` env var to configure jupyter location +- Remove `vscode.conf/exe` configuration in favor of the new, internal default `code-server` installation +- Refactor image to build _FROM_ the new `product-base-pro` image + +# 2022-07-28 + +- Added `conf/launcher.local.conf` with `unprivileged=1` flag so the Local Launcher Plugin starts in unprivileged mode. + +# 2022.07.0 + +- Add a `/usr/local/bin/jupyter` symlink +- Configure Workbench to preinstall R, Python, and Quarto extensions for users on session startup +- Add workbench_jupyterlab python package for the jupyterlab extension + +# 2022-04-07 + +- The Dockerfile now uses BuildKit features and must be built with + DOCKER_BUILDKIT=1. + +# 2021.09.0 + +- **BREAKING**: rename environment variables to use `RSW_` prefix instead of `RSP_` prefix + - i.e. `RSP_LICENSE` is now `RSW_LICENSE` +- **BREAKING**: Change execution model significantly + - Before, we had a single startup script `startup.sh` that would launch multiple services + - There was no service manager, so launcher going down could not be detected + - We are now using `supervisord` as a service manager within the container + - `supervisord` is configured to exit if any of its child processes exit + - `ENTRYPOINT` and `CMD` have changed accordingly + - A more thorough write-up is in `README.md` +- Improve logging behaviors to send logs to `stdout`/`stderr` instead of using `tail` +- Add `sssd` to the container for user provisioning. It starts by default with no directory configured. + - To utilize, mount your own `*.conf` files into `/etc/sssd/conf.d/` (ensure ownership by root, and 0600 permissions) + - Examples and further documentation is in `README.md` + +# 1.4.1717-3 + +- Add `code-server` / vscode installation and configuration +- Added R version 4.1.0. Image includes versions 4.1.0, 3.6.3 +- BREAKING: change image name to `rstudio/rstudio-workbench` +- BREAKING: change the location / version of python. Now includes: + - version 3.8.10 at /opt/python/jupyter to run jupyter + - version 3.8.10 at /opt/python/3.8.10 as a kernel for jupyter + - version 3.9.5 at /opt/python/3.9.5 as a kernel for jupyter + +# 1.4.1106-5 + +- Includes R version 3.6.3 at /opt/R/3.6.3 +- Includes Python version 3.6.5 at /opt/python/3.6.5 diff --git a/workbench-for-google-cloud-workstations/README.md b/workbench-for-google-cloud-workstations/README.md new file mode 100644 index 00000000..6d3f94a4 --- /dev/null +++ b/workbench-for-google-cloud-workstations/README.md @@ -0,0 +1,279 @@ +# Quick reference + +* Maintained by: [the Posit Docker team](https://github.com/rstudio/rstudio-docker-products) +* Where to get help: [our Github Issues page](https://github.com/rstudio/rstudio-docker-products/issues) +* RStudio Workbench image: [Docker Hub](https://hub.docker.com/r/rstudio/rstudio-workbench) +* RStudio r-session-complete image: [Docker Hub](https://hub.docker.com/r/rstudio/r-session-complete) + +# Supported tags and respective Dockerfile links + +* [`2022.07.2`, `bionic`, `ubuntu1804`, `bionic-2022.07.2`, `ubuntu1804-2022.07.2`](https://github.com/rstudio/rstudio-docker-products/blob/main/workbench/Dockerfile.ubuntu1804) +* [`jammy`, `ubuntu2204`, `jammy-2022.07.2`, `ubuntu2204-2022.07.2`](https://github.com/rstudio/rstudio-docker-products/blob/main/workbench/Dockerfile.ubuntu2204) + +# What is RStudio Workbench? + +Posit Workbench, formerly RStudio Workbench, is the preferred data analysis and integrated development experience for +professional R users and data science teams who use R and Python. Posit Workbench enables the collaboration, +centralized management, metrics, security, and commercial support that professional data science teams need to operate +at scale. + +Some of the functionality that Workbench provides is: + +* The ability to develop in Workbench and Jupyter +* Load balancing +* Tutorial API +* Data connectivity and Posit Professional Drivers (formerly RStudio Professional Drivers) +* Collaboration and project sharing +* Scale with Kubernetes and SLURM +* Authentication, access, & security +* Run multiple concurrent R and Python sessions +* Remote execution with Launcher +* Auditing and monitoring +* Advanced R and Python session management + +For more information on running RStudio Workbench in your organization please visit https://www.rstudio.com/products/workbench/. + +# Notice for support + +1. This image may introduce **BREAKING** changes; as such we recommend: + - Avoid using the `{operating-system}` tags to avoid unexpected version changes, and + - Always read through the [NEWS](./NEWS.md) to understand the changes before updating. +1. Outdated images will be removed periodically from DockerHub as product version updates are made. Please make plans to + update at times or use your own build of the images. +1. These images are meant as a starting point for your needs. Consider creating a fork of this repo, where you can + continue to merge in changes we make while having your own security scanning, base OS in use, or other custom + changes. We + provide [instructions for building](https://github.com/rstudio/rstudio-docker-products#instructions-for-building) for + these cases. + +# How to use this image + +To verify basic functionality as a first step: + +``` +# Replace with valid license +export RSW_LICENSE=XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX + +# Run without persistent data using default configuration +docker run -it \ + -p 8787:8787 \ + -e RSW_LICENSE=$RSW_LICENSE \ + rstudio/rstudio-workbench:ubuntu1804 +``` + + +Open [http://localhost:8787](http://localhost:8787) to access RStudio Workbench. The default username and password are +`rstudio`. + +## Overview + +Note that running the RStudio Workbench Docker image requires a valid RStudio Workbench license. + +This container includes: + +1. Two versions of R +2. Two versions of Python +3. RStudio Connect + +## Configuration + +RStudio Workbench is configured via config files in the `/etc/rstudio` directory. Mount this directory as +a volume from the host machine. Changes will take effect when the container is restarted. + +You can review possible RStudio Workbench configuration [in the documentation](https://docs.rstudio.com/ide/workbench/). + +See a complete example of server configuration at `workbench/conf`. + +### Persistent Data + +In order to persist user files between container restarts please mount the `/home` directory from a persistent volume on the host +machine or your docker orchestration system. + +### Licensing + +The RStudio Workbench Docker image requires a valid license, which can be set in three ways: + +1. Setting the `RSW_LICENSE` environment variable to a valid license key inside the container +2. Setting the `RSW_LICENSE_SERVER` environment variable to a valid license server / port inside the container +3. Mounting a `/etc/rstudio-server/license.lic` single file that contains a valid license for RStudio Workbench + +**NOTE:** the "offline activation process" is not supported by this image today. Offline installations will need +to explore using a license server, license file, or custom image with manual intervention. + +### User Provisioning + +By default, the container will create a test user, which you can control or disable with the environment +variables: `RSW_TESTUSER`, `RSW_TESTUSER_PASSWD`, `RSW_TESTUSER_UID`. + +#### sssd / LDAP / Active Directory + +If you have a directory (LDAP server, Active Directory, etc.) available to +provision users, `sssd` is installed in the container and enabled by default ( +see `Process Management` below). In order to make use of it, you will need to +mount your own configuration file into `/etc/sssd/conf.d/`. For instance, + +_sssd.conf_ +```ini +[sssd] +config_file_version = 2 +domains = LDAP + +[domain/LDAP] +id_provider = ldap +auth_provider = ldap +chpass_provider = ldap +sudo_provider = ldap +# ... more configuration +``` + +Then: +```bash +# sssd is picky about file permissions +chmod 600 sssd.conf + +docker run -it \ + -p 8787:8787 -p 5559:5559 \ + -v $PWD/data/rsp:/home \ + -v $PWD/server-pro/conf/:/etc/rstudio \ + -v $PWD/sssd.conf:/etc/sssd/conf.d/sssd.conf \ + -e RSP_LICENSE=$RSP_LICENSE \ + rstudio/rstudio-workbench:ubuntu1804 +``` + +It is worth noting that you may also need to modify the PAM configuration files +in the container, if you are using PAM for custom authentication or session +behavior. See the [RStudio Workbench +guide](https://docs.rstudio.com/ide/server-pro/authenticating-users.html) for +more information. + +### Environment variables + +| Variable | Description | Default | +|-----|---|-----------| +| `RSW_TESTUSER` | Test user to be created in the container, turn off with an empty value | `rstudio` | +| `RSW_TESTUSER_PASSWD` | Test user password | `rstudio` | +| `RSW_TESTUSER_UID` | Test user UID | `10000` | +| `RSW_LICENSE` | License key for RStudio Workbench, format should be: `XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX` | None | +| `RSW_LICENSE_SERVER` | Floating license server, format should be: `my.url.com:port` | None | +| `RSW_LAUNCHER` | Whether or not to use launcher locally / start the launcher process | true | +| `RSW_LAUNCHER_TIMEOUT` | The timeout, in seconds, to wait for launcher to start listening on the expected port before failing startup | 10 | + +### Ports + +| Variable | Description | +|--------|---| +| `8787` | Default HTTP Port for RStudio Connect | +| `5559` | Port for RStudio Launcher server | + +### Example usage: + +```bash +# Replace with valid license +export RSW_LICENSE=XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX + +# Run without persistent data and using an external configuration +docker run -it \ + -p 8787:8787 -p 5559:5559 \ + -v $PWD/workbench/conf/:/etc/rstudio \ + -e RSW_LICENSE=$RSW_LICENSE \ + rstudio/rstudio-workbench:ubuntu1804 + +# Run with persistent data and using an external configuration +docker run -it \ + -p 8787:8787 -p 5559:5559 \ + -v $PWD/data/rsw:/home \ + -v $PWD/workbench/conf/:/etc/rstudio \ + -e RSW_LICENSE=$RSW_LICENSE \ + rstudio/rstudio-workbench:ubuntu1804 +``` + +Open [http://localhost:8787](http://localhost:8787) to access RStudio Workbench. +The default username and password are `rstudio`. + +### Process Management + +In order for RStudio Workbench to function properly, several services need to +be accounted for. We run these services using +[`supervisord`](http://supervisord.org/). `supervisord` is an open source +process supervisor with an active community. It enables running multiple +services in the container, as well as exiting the container if _any_ of those +services exit. + +> NOTE: generally speaking, running multiple services in a single container is an anti-pattern. +> However, we have implemented the below as a workaround until RStudio Workbench can +> handle users and other processes in a more container-friendly fashion + +Details on the various processes and their configuration is below: + +- **RStudio Workbench**: the main server process + - this startup configuration is mounted at `/startup/base` + +- **RStudio Job Launcher**: enables launching Jupyter, JupyterLab, and VSCode + sessions, as well as talking to job schedulers like Slurm and Kubernetes. + - Optional and enabled by default + - this startup configuration is mounted at `/startup/launcher` + - to disable, mount an empty volume over `/startup/launcher` + +- **sssd**: often used for user provisioning when connected to an LDAP + directory or other user store. + - Optional, and enabled by default, but with a "dummy" domain so it does + nothing. + - To use this with your directory, mount required `.conf` files into + `/etc/sssd/conf.d/` (more details in `User Provisioning`, above) + - Startup configuration is installed at `/startup/user-provisioning/` + - To disable entirely, mount an empty volume over + `/startup/user-provisioning/` + +- **custom**: Do you have a service that you need to run inside the container + for user provisioning or otherwise? Mount other configuration files into + `/startup/custom`, and they will be started and managed by `supervisord` as + well + - NOTE: in many cases (i.e. Kubernetes) `initContainers` or `sidecar` containers are a better fit + +## Caveats of product licensing in containers + +*Note: This section **does not** apply to activations using license files.* + +There is currently a known licensing bug when using our products in containers. If the container is not stopped +gracefully, the license deactivation step may fail or be skipped. Failing to deactivate the license can result in a +"license leak" where a product activation is used up and cannot be deactivated using traditional methods as the +activation state on the container has been lost. + +To avoid "leaking" licenses, we encourage users not to force kill containers and to use `--stop-timeout 120` and +`--time 120` for `docker run` and `docker stop` commands respectively. This helps ensure the deactivation script has +ample time to run properly. + +In some situations, it can be difficult or impossible to avoid a hard termination (e.g. power failure, +critical error on host). Unfortunately, any of these cases can still cause a license to leak an activation. To help +prevent a license leak in these situations, users can mount the following directories to persistent storage to preserve +the license state data across restarts of the container. **These directories differ between products.** + +* License Key + * `/var/lib/.local` + * `/var/lib/.prof` + * `/var/lib/rstudio-workbench` +* Floating License + * `/var/lib/.TurboFloat` + +Please note that the files created in these directories are hardware locked and non-transferable between hosts. Due to +the nature of the hardware fingerprinting algorithm, any low-level changes to the host or container can cause existing +license state files to invalidate. To avoid this problem, we advise that product containers are gracefully shutdown +and allowed to deactivate prior to changing any hardware or firmware on the host (e.g. upgrading a network card or +updating BIOS) or the container (e.g. changing the network driver used or the allocated number of CPU cores). + +While preserving license state data *can* help avoid license leaks across restarts, it's not a guarantee. If you run +into issues with your license, please do not hesitate to [contact Posit support](https://support.posit.co/hc/en-us). + +While neither of these solutions will eliminate the problem, they should help mitigate it. We are still investigating a +long-term solution. + +# Licensing + +The license associated with the RStudio Docker Products repository is located [in LICENSE.md](https://github.com/rstudio/rstudio-docker-products/blob/main/LICENSE.md). + +As is the case with all container images, the images themselves also contain other software which may be under other +licenses (i.e. bash, linux, system libraries, etc., along with any other direct or indirect dependencies of the primary +software being contained). + +It is an image user's responsibility to ensure that use of this image (and any of its dependent layers) complies with +all relevant licenses for the software contained in the image. diff --git a/workbench-for-google-cloud-workstations/conf/jupyter.conf b/workbench-for-google-cloud-workstations/conf/jupyter.conf new file mode 100644 index 00000000..abc5cfd5 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/jupyter.conf @@ -0,0 +1,6 @@ +notebooks-enabled=1 +labs-enabled=1 + +# jupyter-exe=/usr/local/bin/jupyter + +default-session-cluster=Local diff --git a/workbench-for-google-cloud-workstations/conf/launcher-env b/workbench-for-google-cloud-workstations/conf/launcher-env new file mode 100644 index 00000000..a1144c18 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/launcher-env @@ -0,0 +1,6 @@ +JobType: session +Environment: LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=en_US.UTF-8 +JobType: any +Environment: PATH=/opt/python/3.8.10/bin:$PATH diff --git a/workbench-for-google-cloud-workstations/conf/launcher.conf b/workbench-for-google-cloud-workstations/conf/launcher.conf new file mode 100644 index 00000000..c73ef787 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/launcher.conf @@ -0,0 +1,12 @@ +[server] +address=127.0.0.1 +port=5559 +server-user=rstudio-server +admin-group=rstudio-server +authorization-enabled=1 +thread-pool-size=4 +enable-debug-logging=1 + +[cluster] +name=Local +type=Local diff --git a/workbench-for-google-cloud-workstations/conf/launcher.local.conf b/workbench-for-google-cloud-workstations/conf/launcher.local.conf new file mode 100644 index 00000000..051857e1 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/launcher.local.conf @@ -0,0 +1 @@ +unprivileged=1 diff --git a/workbench-for-google-cloud-workstations/conf/launcher.pem b/workbench-for-google-cloud-workstations/conf/launcher.pem new file mode 100644 index 00000000..f63cf133 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/launcher.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDAT7k2z06gPhLx +RNLavdwFTHG+KTbJouQeHWjvFY+NFyePINp4r6R+J3z+Wq/LJ4usUvafx4qTTNhL +Iub6Uix+fRbRwuRv8K0y+nOj7nszulRF7Q/3CRWCLoVzNKHiLbbdHma5MpNtH9gz +Cry87PGk8ZQviIt2pFqsI0fiu7wSPdUmMpt2ssBTduOkuNJ8c0aH+UK7uBGQt6yy +q/BLNKiUMc4Krb/1xKxXkzcyhs5OTtfg95fMOItvZtijliLFW/vDnzbGk6IqEonf +ZOwKUB3cLChIGtdkyQ6KzjTRbtzc0D/XVnOuhFsLrYPqoQcap1IqFhhZbzviR1c4 +fOGpnG5lAgMBAAECggEANQfG8y5HaDi10IJCAr/IJE3oL5/906xvaz6A6Szw4N85 +sWZF2HHW0lLhTOKFB2hoQ6S9DWJg/C18l3uQZgay7k5gNIehppyfpicaLGJe5uzC +wGWWuXBv1e4uLo6c9VnVyzPO79LahIT7gVe8o/xSxkE3s8PbcWPw/Pbdo9BRFFYu +Gcm7BxJXvgFW9V1SP1fVT3o93n9vFZa3Ti6uH+XyQqtCasZp4jAmO5aHzDl0zIu8 +iV8l0KdrmoSNuxBOe8tkIU+j1F/orQhX7GJRmHG/vIfOSnonf2wqNGHyeikehzYy +6zjxujN2saYVedP3T1Uuox/4TJUoB8pWRCVjIIbWIQKBgQDqg8QYgC9dJ1Cbl+SX +j8I1lDasPbPjoUvkHxmR6XqQWTIJKai5uuITYMfFRmPxj6vLc7Ot6cR1ndpglX+4 +NM98m99Mquzx6raNunUvZbOZC7U1B2KbXOcH5UWvQWyum9yy78ysfZRxFYCeZErO +0Rvf7vDCfabvVW3jZHRse/4OEwKBgQDR7iKEFbb3gl/Hq/1CVoHbeQk6Ak3f5ZfM +IimYiY86OBZ5qMyF0yoZa8qakR5tmgaIN9ZXIddMEtJF+JVuXA2TCKU13X8REebZ +vXCAirneBmngv6MDzKz3kRN5Q+q2dw2yvBSHzFhF6J9A3ZODJeWuZuiohVoVEH+6 +HGVH4MHApwKBgDROCTmRtX6oWyEo4pIvPRioIv2NkYaunfeWPCTAF1cEndVPVlN+ +bBf4WPYV+gqRibaJ6sRaRDgbae5qyZ6zIKfhOA+oe8qFGAxKTwKwQztaUiMur36J +XPn01/L7u5rCJpo+Q78hDy+dpR8urUStfPMYHtiyH753ple2OwVLLmHDAoGANbIe +fAB940MispToW5566qND2+WY+6rjL+CRGSYeT3rnLMkjPxurCgOLOAvkZ45dG1yE +45aRVjD+Pc1+k1LSWgh9EZENHlWg4ZLWAtr4/bRQQp9h1+fzWgNuorbfgzpic5Qu +ACohbgg5K8NBJut7mydRXUF2sCqfMQBwW3k+tcMCgYAFeSEptO8zfmYI3Q1U84R8 +MvxcSef3GBpL3ZuSIvWeNN3WYKluSn+VxYVUWg2i3/4AuBWqbY6bxdOVJElEcqHS +ALXnZCwMel6A/59Fp86eY4S3Mqj1MMfONwu1h96x3WSxazfzIRdFLTCI5N9J03sR +CXhynMytlSaH5GkThW5N4w== +-----END PRIVATE KEY----- diff --git a/workbench-for-google-cloud-workstations/conf/launcher.pub b/workbench-for-google-cloud-workstations/conf/launcher.pub new file mode 100644 index 00000000..2c5babe0 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/launcher.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwE+5Ns9OoD4S8UTS2r3c +BUxxvik2yaLkHh1o7xWPjRcnjyDaeK+kfid8/lqvyyeLrFL2n8eKk0zYSyLm+lIs +fn0W0cLkb/CtMvpzo+57M7pURe0P9wkVgi6FczSh4i223R5muTKTbR/YMwq8vOzx +pPGUL4iLdqRarCNH4ru8Ej3VJjKbdrLAU3bjpLjSfHNGh/lCu7gRkLessqvwSzSo +lDHOCq2/9cSsV5M3MobOTk7X4PeXzDiLb2bYo5YixVv7w582xpOiKhKJ32TsClAd +3CwoSBrXZMkOis400W7c3NA/11ZzroRbC62D6qEHGqdSKhYYWW874kdXOHzhqZxu +ZQIDAQAB +-----END PUBLIC KEY----- diff --git a/workbench-for-google-cloud-workstations/conf/logging.conf b/workbench-for-google-cloud-workstations/conf/logging.conf new file mode 100644 index 00000000..08a09174 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/logging.conf @@ -0,0 +1,3 @@ +[*] +logger-type=stderr +log-level=info diff --git a/workbench-for-google-cloud-workstations/conf/repos.conf b/workbench-for-google-cloud-workstations/conf/repos.conf new file mode 100644 index 00000000..bee0ac41 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/repos.conf @@ -0,0 +1,2 @@ +RSPM=https://packagemanager.rstudio.com/cran/__linux__/bionic/latest +CRAN=https://packagemanager.rstudio.com/cran/__linux__/bionic/latest diff --git a/workbench-for-google-cloud-workstations/conf/rserver-float.conf b/workbench-for-google-cloud-workstations/conf/rserver-float.conf new file mode 100644 index 00000000..f04dd8c8 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/rserver-float.conf @@ -0,0 +1 @@ +server-license-type=remote diff --git a/workbench-for-google-cloud-workstations/conf/rserver.conf b/workbench-for-google-cloud-workstations/conf/rserver.conf new file mode 100644 index 00000000..2817b52d --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/rserver.conf @@ -0,0 +1,13 @@ +server-health-check-enabled=1 +admin-enabled=1 + +www-port=80 +server-project-sharing=0 +auth-pam-sessions-enabled=1 + +# Launcher Config +launcher-address=127.0.0.1 +launcher-port=5559 +launcher-sessions-enabled=1 +launcher-default-cluster=Local +launcher-sessions-callback-address=http://127.0.0.1:80 diff --git a/workbench-for-google-cloud-workstations/conf/vscode-user-settings.json b/workbench-for-google-cloud-workstations/conf/vscode-user-settings.json new file mode 100644 index 00000000..144b8c2a --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/vscode-user-settings.json @@ -0,0 +1,5 @@ +{ + "terminal.integrated.shell.linux": "/bin/bash", + "extensions.autoUpdate": false, + "extensions.autoCheckUpdates": false +} diff --git a/workbench-for-google-cloud-workstations/conf/vscode.conf b/workbench-for-google-cloud-workstations/conf/vscode.conf new file mode 100644 index 00000000..405ff90e --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/vscode.conf @@ -0,0 +1,4 @@ +enabled=1 +args=--verbose --host=0.0.0.0 + +# exe=/usr/lib/rstudio-server/bin/code-server/bin/code-server diff --git a/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf b/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf new file mode 100644 index 00000000..f528e1a3 --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf @@ -0,0 +1,3 @@ +quarto.quarto +Ikuyadeu.r@2.4.0 +ms-python.python@2022.10.1 diff --git a/workbench-for-google-cloud-workstations/docker-compose.test.yml b/workbench-for-google-cloud-workstations/docker-compose.test.yml new file mode 100644 index 00000000..b28c6f5f --- /dev/null +++ b/workbench-for-google-cloud-workstations/docker-compose.test.yml @@ -0,0 +1,20 @@ +version: '2.3' +services: + + sut: + image: $IMAGE_NAME + command: /run_tests.sh + entrypoint: [] + environment: + # uses .env by default + - RSW_VERSION + - R_VERSION + - PYTHON_VERSION + - PYTHON_VERSION_ALT + - JUPYTER_PYTHON_VERSION + - R_VERSION_ALT + - RSW_LICENSE + - RSW_LICENSE_SERVER + volumes: + - "./test/run_tests.sh:/run_tests.sh" + - "./test/goss.yaml:/tmp/goss.yaml" diff --git a/workbench-for-google-cloud-workstations/pam/rstudio b/workbench-for-google-cloud-workstations/pam/rstudio new file mode 100644 index 00000000..cf173d6d --- /dev/null +++ b/workbench-for-google-cloud-workstations/pam/rstudio @@ -0,0 +1,14 @@ +#%PAM-1.0 +auth sufficient pam_sss.so forward_pass +auth required pam_unix.so try_first_pass nullok +auth optional pam_permit.so +auth required pam_env.so + +account [default=bad success=ok user_unknown=ignore authinfo_unavail=ignore] pam_sss.so +account required pam_unix.so +account optional pam_permit.so +account required pam_time.so + +password sufficient pam_sss.so use_authtok +password required pam_unix.so try_first_pass nullok sha512 shadow +password optional pam_permit.so diff --git a/workbench-for-google-cloud-workstations/pam/rstudio-session b/workbench-for-google-cloud-workstations/pam/rstudio-session new file mode 100644 index 00000000..05894708 --- /dev/null +++ b/workbench-for-google-cloud-workstations/pam/rstudio-session @@ -0,0 +1,26 @@ +#%PAM-1.0 + +# This allows root to su without passwords (this is required) +# DO NOT use in the "rstudio" profile +auth sufficient pam_rootok.so +auth sufficient pam_sss.so forward_pass +auth required pam_unix.so try_first_pass nullok +auth optional pam_permit.so +auth required pam_env.so + +account [default=bad success=ok user_unknown=ignore authinfo_unavail=ignore] pam_sss.so +account required pam_unix.so +account optional pam_permit.so +account required pam_time.so + +password sufficient pam_sss.so use_authtok +password required pam_unix.so try_first_pass nullok sha512 shadow +password optional pam_permit.so + +session required pam_mkhomedir.so skel=/etc/skel umask=0022 +session required pam_env.so readenv=1 +session required pam_env.so readenv=1 envfile=/etc/default/locale +session required pam_limits.so +session required pam_unix.so +session optional pam_sss.so +session optional pam_permit.so diff --git a/workbench-for-google-cloud-workstations/sssd.conf b/workbench-for-google-cloud-workstations/sssd.conf new file mode 100644 index 00000000..7bbba4ab --- /dev/null +++ b/workbench-for-google-cloud-workstations/sssd.conf @@ -0,0 +1,9 @@ +[sssd] +config_file_version = 2 +domains = placeholder + +[domain/placeholder] +id_provider = none +auth_provider = none +chpass_provider = none +sudo_provider = none diff --git a/workbench-for-google-cloud-workstations/startup-launcher/rstudio-launcher.conf b/workbench-for-google-cloud-workstations/startup-launcher/rstudio-launcher.conf new file mode 100644 index 00000000..5fd8540d --- /dev/null +++ b/workbench-for-google-cloud-workstations/startup-launcher/rstudio-launcher.conf @@ -0,0 +1,8 @@ +[program:rstudio-launcher] +command=/usr/lib/rstudio-server/bin/rstudio-launcher +autorestart=false +numprocs=1 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/workbench-for-google-cloud-workstations/startup-user-provisioning/sssd.conf b/workbench-for-google-cloud-workstations/startup-user-provisioning/sssd.conf new file mode 100644 index 00000000..387913e5 --- /dev/null +++ b/workbench-for-google-cloud-workstations/startup-user-provisioning/sssd.conf @@ -0,0 +1,11 @@ +[program:sssd] +# TODO: a way to disable this easily...? +command=/usr/sbin/sssd -i -c /etc/sssd/sssd.conf --logger=stderr +autorestart=false +numprocs=1 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stdout_logfile_backups=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +stderr_logfile_backups=0 diff --git a/workbench-for-google-cloud-workstations/startup.sh b/workbench-for-google-cloud-workstations/startup.sh new file mode 100644 index 00000000..406b7aa0 --- /dev/null +++ b/workbench-for-google-cloud-workstations/startup.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +export LICENSE_MANAGER_PATH=${LICENSE_MANAGER_PATH:-/opt/rstudio-license} + +set -e +set -x + +# Deactivate license when the process exits +deactivate() { + echo "== Exiting ==" + rstudio-server stop + echo "Deactivating license ..." + ${LICENSE_MANAGER_PATH}/license-manager deactivate >/dev/null 2>&1 + + echo "== Done ==" +} +trap deactivate EXIT + +verify_installation(){ + echo "==VERIFY INSTALLATION=="; + mkdir -p $DIAGNOSTIC_DIR + chmod 777 $DIAGNOSTIC_DIR + rstudio-server verify-installation --verify-user=$RSW_TESTUSER | tee $DIAGNOSTIC_DIR/verify.log +} + +# Support RSP_ or RSW_ prefix +RSP_LICENSE=${RSP_LICENSE:-${RSW_LICENSE}} +RSP_LICENSE_SERVER=${RSP_LICENSE_SERVER:-${RSW_LICENSE_SERVER}} + +# Activate License +RSW_LICENSE_FILE_PATH=${RSW_LICENSE_FILE_PATH:-/etc/rstudio-server/license.lic} +if [ -n "$RSP_LICENSE" ]; then + ${LICENSE_MANAGER_PATH}/license-manager activate $RSP_LICENSE || true +elif [ -n "$RSP_LICENSE_SERVER" ]; then + ${LICENSE_MANAGER_PATH}/license-manager license-server $RSP_LICENSE_SERVER || true +elif test -f "$RSW_LICENSE_FILE_PATH"; then + ${LICENSE_MANAGER_PATH}/license-manager activate-file $RSW_LICENSE_FILE_PATH || true +fi + +# ensure these cannot be inherited by child processes +unset RSP_LICENSE +unset RSP_LICENSE_SERVER +unset RSW_LICENSE +unset RSW_LICENSE_SERVER + +# Create one user +if [ $(getent passwd $RSW_TESTUSER_UID) ] ; then + echo "UID $RSW_TESTUSER_UID already exists, not creating $RSW_TESTUSER test user"; +else + if [ -z "$RSW_TESTUSER" ]; then + echo "Empty 'RSW_TESTUSER' variables, not creating test user"; + else + useradd -m -s /bin/bash -N -u $RSW_TESTUSER_UID $RSW_TESTUSER + echo "$RSW_TESTUSER:$RSW_TESTUSER_PASSWD" | sudo chpasswd + fi +fi + +# Start Launcher +if [ "$RSW_LAUNCHER" == "true" ]; then + echo "Waiting for launcher to startup... to disable set RSW_LAUNCHER=false" + wait-for-it.sh localhost:5559 -t $RSW_LAUNCHER_TIMEOUT +fi + +# Check diagnostic configurations +if [ "$DIAGNOSTIC_ENABLE" == "true" ]; then + verify_installation + if [ "$DIAGNOSTIC_ONLY" == "true" ]; then + echo $(<$DIAGNOSTIC_DIR/verify.log); + echo "Exiting script because DIAGNOSTIC_ONLY=${DIAGNOSTIC_ONLY}"; + exit 0 + fi; +else + echo "not running verify installation because DIAGNOSTIC_ENABLE=${DIAGNOSTIC_ENABLE}"; +fi + +# the main container process +# cannot use "exec" or the "trap" will be lost +/usr/lib/rstudio-server/bin/rserver --server-daemonize 0 > /dev/stderr diff --git a/workbench-for-google-cloud-workstations/startup/rstudio-workbench.conf b/workbench-for-google-cloud-workstations/startup/rstudio-workbench.conf new file mode 100644 index 00000000..0acac704 --- /dev/null +++ b/workbench-for-google-cloud-workstations/startup/rstudio-workbench.conf @@ -0,0 +1,8 @@ +[program:rstudio-workbench] +command=/usr/local/bin/startup.sh +autorestart=false +numprocs=1 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/workbench-for-google-cloud-workstations/supervisord.conf b/workbench-for-google-cloud-workstations/supervisord.conf new file mode 100644 index 00000000..7180a826 --- /dev/null +++ b/workbench-for-google-cloud-workstations/supervisord.conf @@ -0,0 +1,47 @@ +; supervisor config file + +[unix_http_server] +file=/var/run/supervisor.sock ; (the path to the socket file) +chmod=0700 ; sockef file mode (default 0700) + +[supervisord] +logfile=/dev/stdout ; (main log file;default $CWD/supervisord.log) +user=root +pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +; should configure each program to use stdout/stderr +; childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) +logfile_maxbytes=0 +logfile_backups=0 +loglevel=info +nodaemon=true + +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +[eventlistener:process-monitor] +command=bash -c "printf 'READY\n' && while read line; do kill -SIGQUIT $PPID; done < /dev/stdin" +events=PROCESS_STATE_STOPPED,PROCESS_STATE_EXITED,PROCESS_STATE_FATAL +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stdout_logfile_backups=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +stderr_logfile_backups=0 + +; beware possible race condition +; if one of these services exit before the process-monitor is up + +[include] +files = /startup/base/*.conf /startup/launcher/*.conf /startup/user-provisioning/*.conf /startup/custom/*.conf diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml new file mode 100644 index 00000000..f6c6b160 --- /dev/null +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -0,0 +1,133 @@ +user: + rstudio-server: + exists: true + uid: 999 + gid: 999 + +group: + rstudio-server: + exists: true + gid: 999 + +package: + rstudio-server: + installed: true + +port: + tcp:80: + listening: true + ip: + - 0.0.0.0 + skip: false + tcp:5559: + listening: true + ip: + - 127.0.0.1 + skip: false + +process: + rserver: + running: true + skip: false + rstudio-launche: + running: true + skip: false + +file: + /etc/rstudio-server/license.lic: + # currently does not exist + # be sure it is NOT a directory + exists: false + /usr/lib/rstudio-server: + exists: true + /opt/R/{{.Env.R_VERSION_ALT}}/bin/R: + exists: true + filetype: file + /opt/python/{{.Env.PYTHON_VERSION}}/bin/python: + exists: true + filetype: symlink + /opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python: + exists: true + filetype: symlink + /opt/python/jupyter/bin/python: + exists: true + filetype: symlink + /opt/python/jupyter/bin/jupyter: + exists: true + /usr/local/bin/jupyter: + exists: true + # this file gets created at startup? + # /var/lib/rstudio-server/r-versions: + # exists: false + /usr/lib/rstudio-server/bin/rserver: + exists: true + /usr/lib/rstudio-server/bin/rstudio-server: + exists: true + /usr/lib/rstudio-server/bin/rstudio-launcher: + exists: true + /var/lib/rstudio-server/monitor/log: + exists: true + owner: rstudio-server + group: rstudio-server + /usr/lib/rstudio-server/bin/code-server/bin/code-server: + exists: true + /etc/rstudio/vscode.conf: + exists: true + /tmp/startup.log: + exists: true + contains: + - "!Error reading /etc/rstudio/rserver.conf:" + /etc/pam.d/common-session: + exists: true + contains: + - "/^session required pam_mkhomedir.so skel=/etc/skel umask=0022$/" + /etc/sssd/sssd.conf: + exists: true + owner: root + group: root + mode: "0600" + +command: + "su rstudio-server -c 'touch /var/lib/rstudio-server/monitor/log/rstudio-server.log'": + title: workbench_monitor_log_file_creates + exit-status: 0 + "touch /var/log/rstudio-server.log": + title: workbench_server_log_creates + exit-status: 0 +# Check product version + "rstudio-server version": + title: workbench_version_matches + exit-status: 0 + stdout: [ + "{{ .Env.RSW_VERSION }}", + "Workbench" + ] + "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": + title: jupyter_works + exit-status: 0 +# Ensure correct python version + "/opt/python/3.8.10/bin/python --version": + title: alternate_python_version_matches + exit-status: 0 + stdout: [ + "{{.Env.PYTHON_VERSION_ALT}}" + ] +# Ensure correct python version + "/opt/python/3.9.5/bin/python --version": + title: python_version_matches + exit-status: 0 + stdout: [ + "{{.Env.PYTHON_VERSION}}" + ] + "/opt/python/jupyter/bin/python --version": + title: jupyter_python_version_matches + exit-status: 0 + stdout: [ + "{{.Env.JUPYTER_PYTHON_VERSION}}" + ] + "which openssl": + title: uses_system_openssl + exit-status: 0 + stdout: [ + "/usr/bin/openssl" + ] diff --git a/workbench-for-google-cloud-workstations/test/run_tests.sh b/workbench-for-google-cloud-workstations/test/run_tests.sh new file mode 100755 index 00000000..9dc299cb --- /dev/null +++ b/workbench-for-google-cloud-workstations/test/run_tests.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -xe + +RSW_TIMEOUT=${RSW_TIMEOUT:-15} + +touch /tmp/startup.log +trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/startup.log; exit $err' ERR + +# start rstudio-server +echo "--> Starting RStudio Workbench" +/usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & + +echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" +wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT +echo "--> Startup complete" + +GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} +GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} +GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} + +# default to empty var file (since vars are not necessary) +if [ ! -f "$GOSS_VARS" ]; then + touch $GOSS_VARS +fi + +# install goss to tmp location and make executable +curl -sL https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /tmp/goss \ + && chmod +x /tmp/goss \ + && GOSS=/tmp/goss + +GOSS_FILE=$GOSS_FILE GOSS_VARS=$GOSS_VARS $GOSS v --format documentation --max-concurrent $GOSS_MAX_CONCURRENT From 6e248942aa9810a5212b65a18328df266d59b1ef Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 8 Mar 2023 13:04:01 -0800 Subject: [PATCH 02/86] Add changes pulled from Cole's alpha image --- workbench-for-google-cloud-workstations/Dockerfile | 3 ++- workbench-for-google-cloud-workstations/conf/rserver.conf | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 84cc4296..755ce523 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -58,7 +58,8 @@ ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64.asc /tini. RUN gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \ && gpg --batch --verify /tini.asc /tini \ && chmod +x /tini \ - && ln -s /tini /usr/local/bin/tini + && ln -s /tini /usr/local/bin/tini \ + && rm /tini.asc ### Install R versions ### RUN curl -O https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION}_1_amd64.deb \ diff --git a/workbench-for-google-cloud-workstations/conf/rserver.conf b/workbench-for-google-cloud-workstations/conf/rserver.conf index 2817b52d..5bfc7bc1 100644 --- a/workbench-for-google-cloud-workstations/conf/rserver.conf +++ b/workbench-for-google-cloud-workstations/conf/rserver.conf @@ -10,4 +10,7 @@ launcher-address=127.0.0.1 launcher-port=5559 launcher-sessions-enabled=1 launcher-default-cluster=Local -launcher-sessions-callback-address=http://127.0.0.1:80 +launcher-sessions-callback-address=http://127.0.0.1:8787 +auth-proxy=1 +auth-proxy-sign-in-url=http://localhost:80/ +auth-proxy-user-header=x-custom-user-name From 644ba11b297302fb8acf774d8cc293f8fa9e9b25 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 14 Mar 2023 11:03:06 -0700 Subject: [PATCH 03/86] Changes necessary for working image --- .../Dockerfile | 31 +++++++------------ .../Justfile | 10 +++--- .../conf/launcher.conf | 1 - .../conf/nginx.site.conf | 2 ++ .../startup.sh | 19 ++---------- .../120_start-workbench.sh | 5 +++ 6 files changed, 26 insertions(+), 42 deletions(-) create mode 100644 workbench-for-google-cloud-workstations/conf/nginx.site.conf create mode 100755 workbench-for-google-cloud-workstations/workstation-startup/120_start-workbench.sh diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 755ce523..f8f157e2 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -2,13 +2,12 @@ FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 -ARG PYTHON_VERSION_JUPYTER=3.8.10 -ARG TINI_VERSION=0.19.0 -ARG QUARTO_VERSION=1.1.251 +ARG R_VERSION=4.2.2 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.10.10 +ARG PYTHON_VERSION_ALT=3.9.16 +ARG PYTHON_VERSION_JUPYTER=3.9.16 +ARG QUARTO_VERSION=1.2.335 ARG DRIVERS_VERSION=2022.11.0 ARG RSW_VERSION=2022.12.0+353.pro20 ARG RSW_NAME=rstudio-workbench @@ -16,7 +15,7 @@ ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/debian-11/amd64 ENV RSW_LICENSE "" ENV RSW_LICENSE_SERVER "" -ENV RSW_TESTUSER rstudio +ENV RSW_TESTUSER user ENV RSW_TESTUSER_PASSWD rstudio ENV RSW_TESTUSER_UID 10000 ENV RSW_LAUNCHER true @@ -24,6 +23,7 @@ ENV RSW_LAUNCHER_TIMEOUT 10 ENV DIAGNOSTIC_DIR /var/log/rstudio ENV DIAGNOSTIC_ENABLE false ENV DIAGNOSTIC_ONLY false +ENV LICENSE_MANAGER_PATH /usr/lib/rstudio-server/bin ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter @@ -52,15 +52,6 @@ RUN apt-get update --fix-missing \ supervisor \ && rm -rf /var/lib/apt/lists/* -### Install tini ### -ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64 /tini -ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64.asc /tini.asc -RUN gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \ - && gpg --batch --verify /tini.asc /tini \ - && chmod +x /tini \ - && ln -s /tini /usr/local/bin/tini \ - && rm /tini.asc - ### Install R versions ### RUN curl -O https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION}_1_amd64.deb \ && curl -O https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ @@ -141,12 +132,15 @@ RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/r COPY --chmod=600 sssd.conf /etc/sssd/sssd.conf COPY conf/* /etc/rstudio/ +COPY --chmod=600 conf/launcher.p* /etc/rstudio RUN mkdir -p /var/lib/rstudio-server/monitor/log \ && chown -R rstudio-server:rstudio-server /var/lib/rstudio-server/monitor \ && mkdir -p /startup/custom/ \ && printf '\n# allow home directory creation\nsession required pam_mkhomedir.so skel=/etc/skel umask=0022' >> /etc/pam.d/common-session +COPY --chmod=755 workstation-startup/* /etc/workstation-startup.d/ + ### Clean up ### RUN apt-get remove -yq dpkg-sig \ && apt-get install -yqf --no-install-recommends \ @@ -156,6 +150,3 @@ RUN apt-get remove -yq dpkg-sig \ EXPOSE 80/tcp EXPOSE 5559/tcp - -ENTRYPOINT [] -CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 9644785c..3d8117f5 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -11,11 +11,11 @@ RSW_LICENSE := "" RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" -R_VERSION := "3.6.2" -R_VERSION_ALT := "4.1.0" +R_VERSION := "4.2.2" +R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" +PYTHON_VERSION := "3.10.10" +PYTHON_VERSION_ALT := "3.9.16" _make-default-tag: echo "{{IMAGE_PREFIX}}{{PRODUCT}}:$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" @@ -80,7 +80,7 @@ run TAG=`just _make-default-tag` CMD="": docker run -it --privileged \ ${volume_opts[@]} \ - -p 80:8787 \ + -p 8787:80 \ -e RSW_LICENSE="{{ RSW_LICENSE }}" \ -e RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ "{{ TAG }}" {{ CMD }} diff --git a/workbench-for-google-cloud-workstations/conf/launcher.conf b/workbench-for-google-cloud-workstations/conf/launcher.conf index c73ef787..dc48bd19 100644 --- a/workbench-for-google-cloud-workstations/conf/launcher.conf +++ b/workbench-for-google-cloud-workstations/conf/launcher.conf @@ -4,7 +4,6 @@ port=5559 server-user=rstudio-server admin-group=rstudio-server authorization-enabled=1 -thread-pool-size=4 enable-debug-logging=1 [cluster] diff --git a/workbench-for-google-cloud-workstations/conf/nginx.site.conf b/workbench-for-google-cloud-workstations/conf/nginx.site.conf new file mode 100644 index 00000000..2d302a7d --- /dev/null +++ b/workbench-for-google-cloud-workstations/conf/nginx.site.conf @@ -0,0 +1,2 @@ +proxy_set_header X-CUSTOM-USER-NAME user/google; +proxy_set_header Host $http_x_forwarded_host; \ No newline at end of file diff --git a/workbench-for-google-cloud-workstations/startup.sh b/workbench-for-google-cloud-workstations/startup.sh index 406b7aa0..76d3b19f 100644 --- a/workbench-for-google-cloud-workstations/startup.sh +++ b/workbench-for-google-cloud-workstations/startup.sh @@ -1,9 +1,8 @@ #!/bin/bash -export LICENSE_MANAGER_PATH=${LICENSE_MANAGER_PATH:-/opt/rstudio-license} +export LICENSE_MANAGER_PATH=${LICENSE_MANAGER_PATH:-/usr/lib/rstudio-server/bin} -set -e -set -x +set -ex # Deactivate license when the process exits deactivate() { @@ -20,7 +19,7 @@ verify_installation(){ echo "==VERIFY INSTALLATION=="; mkdir -p $DIAGNOSTIC_DIR chmod 777 $DIAGNOSTIC_DIR - rstudio-server verify-installation --verify-user=$RSW_TESTUSER | tee $DIAGNOSTIC_DIR/verify.log + rstudio-server verify-installation --verify-user=$RSW_TESTUSER | tee $DIAGNOSTIC_DIR/verify.log } # Support RSP_ or RSW_ prefix @@ -43,18 +42,6 @@ unset RSP_LICENSE_SERVER unset RSW_LICENSE unset RSW_LICENSE_SERVER -# Create one user -if [ $(getent passwd $RSW_TESTUSER_UID) ] ; then - echo "UID $RSW_TESTUSER_UID already exists, not creating $RSW_TESTUSER test user"; -else - if [ -z "$RSW_TESTUSER" ]; then - echo "Empty 'RSW_TESTUSER' variables, not creating test user"; - else - useradd -m -s /bin/bash -N -u $RSW_TESTUSER_UID $RSW_TESTUSER - echo "$RSW_TESTUSER:$RSW_TESTUSER_PASSWD" | sudo chpasswd - fi -fi - # Start Launcher if [ "$RSW_LAUNCHER" == "true" ]; then echo "Waiting for launcher to startup... to disable set RSW_LAUNCHER=false" diff --git a/workbench-for-google-cloud-workstations/workstation-startup/120_start-workbench.sh b/workbench-for-google-cloud-workstations/workstation-startup/120_start-workbench.sh new file mode 100755 index 00000000..ad889455 --- /dev/null +++ b/workbench-for-google-cloud-workstations/workstation-startup/120_start-workbench.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -ex + +/usr/bin/supervisord -c /etc/supervisor/supervisord.conf From a4ee3cb293da5d53954eb6f3c90bd53ea4d889b7 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 29 Mar 2023 11:18:59 -0700 Subject: [PATCH 04/86] Add VS Code "cloudcode" plugin --- workbench-for-google-cloud-workstations/Dockerfile | 1 + .../conf/vscode.extensions.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index f8f157e2..7a8b2572 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update --fix-missing \ && apt-get upgrade -yq \ && apt-get install -yq --no-install-recommends \ dpkg-sig \ + gnupg \ krb5-user \ libcap2 \ libcurl4-gnutls-dev \ diff --git a/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf b/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf index f528e1a3..9fe91992 100644 --- a/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf +++ b/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf @@ -1,3 +1,4 @@ quarto.quarto +GoogleCloudTools.cloudcode Ikuyadeu.r@2.4.0 ms-python.python@2022.10.1 From 615fc8a0fc6bd6c61cb7c8c7f2c7b73d550653f2 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 7 Apr 2023 07:30:46 -0700 Subject: [PATCH 05/86] Upgrade to Workbench 2023.03.0 --- workbench-for-google-cloud-workstations/Dockerfile | 2 +- workbench-for-google-cloud-workstations/Justfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 7a8b2572..f0231621 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -9,7 +9,7 @@ ARG PYTHON_VERSION_ALT=3.9.16 ARG PYTHON_VERSION_JUPYTER=3.9.16 ARG QUARTO_VERSION=1.2.335 ARG DRIVERS_VERSION=2022.11.0 -ARG RSW_VERSION=2022.12.0+353.pro20 +ARG RSW_VERSION=2023.03.0+386.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/debian-11/amd64 diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 3d8117f5..de8f49d6 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -6,7 +6,7 @@ IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu1804" -RSW_VERSION := "2022.12.0+353.pro20" +RSW_VERSION := "2023.03.0+386.pro1" RSW_LICENSE := "" RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" From a4a10502154853fd9e0b182d60e55956d5e85a7a Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 11:00:27 -0700 Subject: [PATCH 06/86] Build workflow and Jupyter improvements --- .../workflows/build-workbench-for-gcw.yaml | 73 +++++++++++++++++++ .../Dockerfile | 5 +- .../conf/jupyter.conf | 2 +- 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build-workbench-for-gcw.yaml diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml new file mode 100644 index 00000000..66607b6b --- /dev/null +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -0,0 +1,73 @@ +on: + schedule: + - cron: "0 8 * * 1" + push: + branches: + - gcw-workbench + +name: Build Workbench for Google Cloud Workstations +jobs: + build-workbench-for-gcw: + runs-on: ubuntu-latest + env: + R_VERSION: 4.2.2 + R_VERSION_ALT: 4.1.3 + PYTHON_VERSION: 3.10.10 + PYTHON_VERSION_ALT: 3.9.16 + + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + ref: gcw-workbench + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v1 + with: + credentials_json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Get Version + id: get-version + run: | + VERSION=`just -f ci.Justfile get-version workbench --type=release --local` + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + + - name: Build Image + uses: docker/build-push-action@v4 + with: + push: false + context: ./workbench-for-google-cloud-workstations + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + build-args: + RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} + tags: + - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest + - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.VERSION }} + + - name: Test Image + env: + IMAGE_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest + RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} + R_VERSION: $R_VERSION + R_VERSION_ALT: $R_VERSION_ALT + PYTHON_VERSION: $PYTHON_VERSION + PYTHON_VERSION_ALT: $PYTHON_VERSION_ALT + run: + docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut + + - name: Push Image + run: | + gcloud auth configure-docker us-central1-docker.pkg.dev + docker push -a us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index f0231621..9e39433b 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -6,7 +6,7 @@ ARG R_VERSION=4.2.2 ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.10.10 ARG PYTHON_VERSION_ALT=3.9.16 -ARG PYTHON_VERSION_JUPYTER=3.9.16 +ARG PYTHON_VERSION_JUPYTER=3.10.10 ARG QUARTO_VERSION=1.2.335 ARG DRIVERS_VERSION=2022.11.0 ARG RSW_VERSION=2023.03.0+386.pro1 @@ -126,7 +126,8 @@ RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyt && /opt/python/jupyter/bin/jupyter-nbextension enable --sys-prefix --py rsp_jupyter \ && /opt/python/jupyter/bin/jupyter-nbextension install --sys-prefix --py rsconnect_jupyter \ && /opt/python/jupyter/bin/jupyter-nbextension enable --sys-prefix --py rsconnect_jupyter \ - && /opt/python/jupyter/bin/jupyter-serverextension enable --sys-prefix --py rsconnect_jupyter + && /opt/python/jupyter/bin/jupyter-serverextension enable --sys-prefix --py rsconnect_jupyter \ + && /opt/python/jupyter/bin/python -m ipykernel install --name py${PYTHON_VERSION_ALT} --display-name "Python ${PYTHON_VERSION_ALT}" RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ && chmod +x /usr/local/bin/wait-for-it.sh diff --git a/workbench-for-google-cloud-workstations/conf/jupyter.conf b/workbench-for-google-cloud-workstations/conf/jupyter.conf index abc5cfd5..1727fbc4 100644 --- a/workbench-for-google-cloud-workstations/conf/jupyter.conf +++ b/workbench-for-google-cloud-workstations/conf/jupyter.conf @@ -1,6 +1,6 @@ notebooks-enabled=1 labs-enabled=1 -# jupyter-exe=/usr/local/bin/jupyter +jupyter-exe=/usr/local/bin/jupyter default-session-cluster=Local From d18d1f4d133f9934ba387f52f7ec2366544f5ce9 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 11:02:06 -0700 Subject: [PATCH 07/86] Pipeline syntax fix --- .github/workflows/build-workbench-for-gcw.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 66607b6b..8d877f30 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -18,7 +18,8 @@ jobs: steps: - name: Check Out Repo uses: actions/checkout@v3 - ref: gcw-workbench + with: + ref: gcw-workbench - name: Authenticate to Google Cloud uses: google-github-actions/auth@v1 From 59c5311a49ee17ee88a618878f6811ea39aa716c Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 11:03:25 -0700 Subject: [PATCH 08/86] Fix build-args syntax --- .github/workflows/build-workbench-for-gcw.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 8d877f30..9002698f 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -52,7 +52,11 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache build-args: - RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} + - RSW_VERSION=${{ steps.get-version.outputs.VERSION }} + - R_VERSION=$R_VERSION + - R_VERSION_ALT=$R_VERSION_ALT + - PYTHON_VERSION=$PYTHON_VERSION + - PYTHON_VERSION_ALT=$PYTHON_VERSION_ALT tags: - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.VERSION }} From c1c0c9b555ac9308e57f979a906f383043aa2594 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 11:06:41 -0700 Subject: [PATCH 09/86] One-line tags --- .github/workflows/build-workbench-for-gcw.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 9002698f..5d43849d 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -57,9 +57,7 @@ jobs: - R_VERSION_ALT=$R_VERSION_ALT - PYTHON_VERSION=$PYTHON_VERSION - PYTHON_VERSION_ALT=$PYTHON_VERSION_ALT - tags: - - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest - - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.VERSION }} + tags: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest,us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.VERSION }} - name: Test Image env: From 1e37205b1e364dc3f824a37146ea86ac2e5b9424 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 11:07:58 -0700 Subject: [PATCH 10/86] Use dumb docker list format --- .github/workflows/build-workbench-for-gcw.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 5d43849d..b3cc5435 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -51,13 +51,15 @@ jobs: context: ./workbench-for-google-cloud-workstations cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - build-args: - - RSW_VERSION=${{ steps.get-version.outputs.VERSION }} - - R_VERSION=$R_VERSION - - R_VERSION_ALT=$R_VERSION_ALT - - PYTHON_VERSION=$PYTHON_VERSION - - PYTHON_VERSION_ALT=$PYTHON_VERSION_ALT - tags: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest,us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.VERSION }} + build-args: | + RSW_VERSION=${{ steps.get-version.outputs.VERSION }} + R_VERSION=$R_VERSION + R_VERSION_ALT=$R_VERSION_ALT + PYTHON_VERSION=$PYTHON_VERSION + PYTHON_VERSION_ALT=$PYTHON_VERSION_ALT + tags: | + us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, + us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.VERSION }} - name: Test Image env: From 3f3aae5babd225a6836604ac47ac3b5ce858e50b Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 11:09:05 -0700 Subject: [PATCH 11/86] Add just setup --- .github/workflows/build-workbench-for-gcw.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index b3cc5435..3edb6ffd 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -26,6 +26,11 @@ jobs: with: credentials_json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 From ab7fc3027a52b7fac205390e55ef81a3ee47eba6 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 11:12:45 -0700 Subject: [PATCH 12/86] Trim double quotes --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 3edb6ffd..ce28d83b 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -47,7 +47,7 @@ jobs: id: get-version run: | VERSION=`just -f ci.Justfile get-version workbench --type=release --local` - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "VERSION=${VERSION//\"/}" >> $GITHUB_OUTPUT - name: Build Image uses: docker/build-push-action@v4 From ddddc938e495148b50ac9a8df47e1986478a0756 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 11:17:09 -0700 Subject: [PATCH 13/86] Fix + in tag version --- .github/workflows/build-workbench-for-gcw.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index ce28d83b..33de178d 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -47,7 +47,10 @@ jobs: id: get-version run: | VERSION=`just -f ci.Justfile get-version workbench --type=release --local` - echo "VERSION=${VERSION//\"/}" >> $GITHUB_OUTPUT + VERSION=${VERSION//\"/} + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "TAG_SAFE_VERSION=${VERSION/+/-}" >> $GITHUB_OUTPUT + - name: Build Image uses: docker/build-push-action@v4 @@ -64,7 +67,7 @@ jobs: PYTHON_VERSION_ALT=$PYTHON_VERSION_ALT tags: | us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.VERSION }} + us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.TAG_SAFE_VERSION }} - name: Test Image env: From aae928e4ead76b6ce02bd8c6392ff1f5978bee41 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 12:54:11 -0700 Subject: [PATCH 14/86] Adapt Dockerfile to Ubuntu 20.04 base --- workbench-for-google-cloud-workstations/Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 9e39433b..463137d4 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -11,7 +11,7 @@ ARG QUARTO_VERSION=1.2.335 ARG DRIVERS_VERSION=2022.11.0 ARG RSW_VERSION=2023.03.0+386.pro1 ARG RSW_NAME=rstudio-workbench -ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/debian-11/amd64 +ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 ENV RSW_LICENSE "" ENV RSW_LICENSE_SERVER "" @@ -37,6 +37,7 @@ RUN apt-get update --fix-missing \ libcap2 \ libcurl4-gnutls-dev \ libglib2.0-0 \ + libicu-dev \ libnss-sss \ libpam-sss \ libpq-dev \ @@ -54,8 +55,8 @@ RUN apt-get update --fix-missing \ && rm -rf /var/lib/apt/lists/* ### Install R versions ### -RUN curl -O https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ +RUN curl -O https://cdn.rstudio.com/r/ubuntu-2004/pkgs/r-${R_VERSION}_1_amd64.deb \ + && curl -O https://cdn.rstudio.com/r/ubuntu-2004/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ && apt-get update \ && apt-get install -yq --no-install-recommends ./r-${R_VERSION}_1_amd64.deb \ && apt-get install -yq --no-install-recommends ./r-${R_VERSION_ALT}_1_amd64.deb \ @@ -65,8 +66,8 @@ RUN curl -O https://cdn.rstudio.com/r/debian-11/pkgs/r-${R_VERSION}_1_amd64.deb && ln -s /opt/R/${R_VERSION}/bin/Rscript /usr/local/bin/Rscript ### Install Python versions ### -RUN curl -O https://cdn.rstudio.com/python/debian-11/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/python/debian-11/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \ +RUN curl -O https://cdn.rstudio.com/python/ubuntu-2004/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ + && curl -O https://cdn.rstudio.com/python/ubuntu-2004/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \ && apt-get update \ && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION}_1_amd64.deb \ && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \ From 6eebba8544a313c3f3ebcd4c477309a1a70c92bd Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 13:03:57 -0700 Subject: [PATCH 15/86] Fix env var references --- .github/workflows/build-workbench-for-gcw.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 33de178d..21dfa16d 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -61,10 +61,10 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache build-args: | RSW_VERSION=${{ steps.get-version.outputs.VERSION }} - R_VERSION=$R_VERSION - R_VERSION_ALT=$R_VERSION_ALT - PYTHON_VERSION=$PYTHON_VERSION - PYTHON_VERSION_ALT=$PYTHON_VERSION_ALT + R_VERSION=${{ env.R_VERSION }} + R_VERSION_ALT=${{ env.R_VERSION_ALT }} + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} tags: | us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.TAG_SAFE_VERSION }} @@ -73,10 +73,10 @@ jobs: env: IMAGE_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} - R_VERSION: $R_VERSION - R_VERSION_ALT: $R_VERSION_ALT - PYTHON_VERSION: $PYTHON_VERSION - PYTHON_VERSION_ALT: $PYTHON_VERSION_ALT + R_VERSION: ${{ env.R_VERSION }} + R_VERSION_ALT: ${{ env.R_VERSION_ALT }} + PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} run: docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut From 92c8ae62bbe771f2b597727caaf7cb09c4796c34 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 13:41:33 -0700 Subject: [PATCH 16/86] Fix odbc install OS --- workbench-for-google-cloud-workstations/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 463137d4..fb2c39d4 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -93,7 +93,7 @@ RUN apt-get update \ && rm -f ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ && rm -rf /var/lib/apt/lists/* \ && cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \ - && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/debian-11/latest")' + && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/ubuntu-2004/latest")' ### Install Workbench ### SHELL ["/bin/bash", "-o", "pipefail", "-c"] From 3044a2a393c4d4d6c3e0184b1ac5a3da6f75056b Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 10 Apr 2023 13:57:07 -0700 Subject: [PATCH 17/86] Fix download URL --- workbench-for-google-cloud-workstations/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index fb2c39d4..06856bf9 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -11,7 +11,7 @@ ARG QUARTO_VERSION=1.2.335 ARG DRIVERS_VERSION=2022.11.0 ARG RSW_VERSION=2023.03.0+386.pro1 ARG RSW_NAME=rstudio-workbench -ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 +ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 ENV RSW_LICENSE "" ENV RSW_LICENSE_SERVER "" @@ -93,7 +93,7 @@ RUN apt-get update \ && rm -f ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ && rm -rf /var/lib/apt/lists/* \ && cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \ - && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/ubuntu-2004/latest")' + && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/focal/latest")' ### Install Workbench ### SHELL ["/bin/bash", "-o", "pipefail", "-c"] From 5255855f02cc6d8182bef357c19394a539f6c9d7 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 11 Apr 2023 12:33:31 -0700 Subject: [PATCH 18/86] Disable build tests for now until I get Goss working again --- .../workflows/build-workbench-for-gcw.yaml | 20 +++++++++---------- .../Dockerfile | 5 +++++ .../conf/rserver.conf | 2 +- .../test/run_tests.sh | 5 +++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 21dfa16d..1e62c5ca 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -69,16 +69,16 @@ jobs: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.TAG_SAFE_VERSION }} - - name: Test Image - env: - IMAGE_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest - RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} - R_VERSION: ${{ env.R_VERSION }} - R_VERSION_ALT: ${{ env.R_VERSION_ALT }} - PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} - run: - docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut +# - name: Test Image +# env: +# IMAGE_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest +# RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} +# R_VERSION: ${{ env.R_VERSION }} +# R_VERSION_ALT: ${{ env.R_VERSION_ALT }} +# PYTHON_VERSION: ${{ env.PYTHON_VERSION }} +# PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} +# run: +# docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut - name: Push Image run: | diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 06856bf9..e731f653 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -31,6 +31,10 @@ ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter RUN apt-get update --fix-missing \ && apt-get upgrade -yq \ && apt-get install -yq --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + dirmngr \ dpkg-sig \ gnupg \ krb5-user \ @@ -43,6 +47,7 @@ RUN apt-get update --fix-missing \ libpq-dev \ libpq5 \ libsm6 \ + libssl-dev \ libuser \ libuser1-dev \ libxext6 \ diff --git a/workbench-for-google-cloud-workstations/conf/rserver.conf b/workbench-for-google-cloud-workstations/conf/rserver.conf index 5bfc7bc1..c9aca81c 100644 --- a/workbench-for-google-cloud-workstations/conf/rserver.conf +++ b/workbench-for-google-cloud-workstations/conf/rserver.conf @@ -10,7 +10,7 @@ launcher-address=127.0.0.1 launcher-port=5559 launcher-sessions-enabled=1 launcher-default-cluster=Local -launcher-sessions-callback-address=http://127.0.0.1:8787 +launcher-sessions-callback-address=http://127.0.0.1:80 auth-proxy=1 auth-proxy-sign-in-url=http://localhost:80/ auth-proxy-user-header=x-custom-user-name diff --git a/workbench-for-google-cloud-workstations/test/run_tests.sh b/workbench-for-google-cloud-workstations/test/run_tests.sh index 9dc299cb..c8d8e170 100755 --- a/workbench-for-google-cloud-workstations/test/run_tests.sh +++ b/workbench-for-google-cloud-workstations/test/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash set -xe -RSW_TIMEOUT=${RSW_TIMEOUT:-15} +RSW_TIMEOUT=${RSW_TIMEOUT:-60} touch /tmp/startup.log trap 'err=$?; echo >&2 "run_tests.sh encountered an error: $err"; cat /tmp/startup.log; exit $err' ERR @@ -11,7 +11,8 @@ echo "--> Starting RStudio Workbench" /usr/bin/supervisord -c /etc/supervisor/supervisord.conf > /tmp/startup.log 2>&1 & echo "--> Waiting for workbench to startup... with RSW_TIMEOUT: $RSW_TIMEOUT" -wait-for-it.sh localhost:8787 -t $RSW_TIMEOUT +wait-for-it.sh localhost:80 -t $RSW_TIMEOUT +wait-for-it.sh localhost:5559 -t $RSW_TIMEOUT echo "--> Startup complete" GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} From 6603b4ced12f93aff2d4094bcd15cbb019cd16f1 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 11 Apr 2023 12:49:21 -0700 Subject: [PATCH 19/86] Quiet flag on gcloud auth --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 1e62c5ca..22a42ef0 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -82,5 +82,5 @@ jobs: - name: Push Image run: | - gcloud auth configure-docker us-central1-docker.pkg.dev + gcloud auth configure-docker -q us-central1-docker.pkg.dev docker push -a us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw From 9284a6cba70adc5cf943c59d514780474d15079a Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 11 Apr 2023 13:03:05 -0700 Subject: [PATCH 20/86] Merge push action into build step --- .github/workflows/build-workbench-for-gcw.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 22a42ef0..6c2d92c2 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -50,12 +50,15 @@ jobs: VERSION=${VERSION//\"/} echo "VERSION=$VERSION" >> $GITHUB_OUTPUT echo "TAG_SAFE_VERSION=${VERSION/+/-}" >> $GITHUB_OUTPUT - + + - name: Authenticate gcloud + run: | + gcloud auth configure-docker -q us-central1-docker.pkg.dev - name: Build Image uses: docker/build-push-action@v4 with: - push: false + push: true context: ./workbench-for-google-cloud-workstations cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache @@ -79,8 +82,3 @@ jobs: # PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} # run: # docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut - - - name: Push Image - run: | - gcloud auth configure-docker -q us-central1-docker.pkg.dev - docker push -a us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw From 688a26c496fcac15ef442178dfab6d6964378735 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 12 Apr 2023 07:26:33 -0700 Subject: [PATCH 21/86] Add configure script step for jupyter notebook --- workbench-for-google-cloud-workstations/conf/rserver.conf | 2 +- .../jupyter/jupyter_notebook_config.py | 1 + .../workstation-startup/110_config-jupyter.sh | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.py create mode 100755 workbench-for-google-cloud-workstations/workstation-startup/110_config-jupyter.sh diff --git a/workbench-for-google-cloud-workstations/conf/rserver.conf b/workbench-for-google-cloud-workstations/conf/rserver.conf index c9aca81c..ddf0ba60 100644 --- a/workbench-for-google-cloud-workstations/conf/rserver.conf +++ b/workbench-for-google-cloud-workstations/conf/rserver.conf @@ -10,7 +10,7 @@ launcher-address=127.0.0.1 launcher-port=5559 launcher-sessions-enabled=1 launcher-default-cluster=Local -launcher-sessions-callback-address=http://127.0.0.1:80 +launcher-sessions-callback-address=http://0.0.0.0:80 auth-proxy=1 auth-proxy-sign-in-url=http://localhost:80/ auth-proxy-user-header=x-custom-user-name diff --git a/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.py b/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.py new file mode 100644 index 00000000..93aea317 --- /dev/null +++ b/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.py @@ -0,0 +1 @@ +c.ServerApp.terminado_settings = {'shell_command': ['/bin/bash']} diff --git a/workbench-for-google-cloud-workstations/workstation-startup/110_config-jupyter.sh b/workbench-for-google-cloud-workstations/workstation-startup/110_config-jupyter.sh new file mode 100755 index 00000000..6f97b81a --- /dev/null +++ b/workbench-for-google-cloud-workstations/workstation-startup/110_config-jupyter.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +mkdir -p /home/user/.jupyter +echo "c.ServerApp.terminado_settings = {'shell_command': ['/bin/bash']}" > /home/user/.jupyter/jupyter_notebook_config.py +chown -R user:user /home/user/.jupyter +chmod 644 /home/user/.jupyter/jupyter_notebook_config.py From 384f8a7ee07fbe565c4e7a3c74f6a53aa5da9281 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 12 Apr 2023 10:29:13 -0700 Subject: [PATCH 22/86] Add jupyter notebook config to fix jupyter terminal bug --- .../Dockerfile | 1 + .../jupyter/jupyter_notebook_config.json | 15 +++++++++++++++ .../jupyter/jupyter_notebook_config.py | 1 - 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.json delete mode 100644 workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.py diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index e731f653..c28b0004 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -148,6 +148,7 @@ RUN mkdir -p /var/lib/rstudio-server/monitor/log \ && printf '\n# allow home directory creation\nsession required pam_mkhomedir.so skel=/etc/skel umask=0022' >> /etc/pam.d/common-session COPY --chmod=755 workstation-startup/* /etc/workstation-startup.d/ +COPY --chmod=644 jupyter/jupyter_notebook_config.json /opt/python/jupyter/etc/jupyter/jupyter_notebook_config.json ### Clean up ### RUN apt-get remove -yq dpkg-sig \ diff --git a/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.json b/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.json new file mode 100644 index 00000000..aec0a83e --- /dev/null +++ b/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.json @@ -0,0 +1,15 @@ +{ + "NotebookApp": { + "nbserver_extensions":{ + "rsconnect_jupyter": true + }, + "terminado_settings": { + "shell_command": ["/bin/bash"] + } + }, + "ServerApp": { + "terminado_settings": { + "shell_command": ["/bin/bash"] + } + } +} \ No newline at end of file diff --git a/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.py b/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.py deleted file mode 100644 index 93aea317..00000000 --- a/workbench-for-google-cloud-workstations/jupyter/jupyter_notebook_config.py +++ /dev/null @@ -1 +0,0 @@ -c.ServerApp.terminado_settings = {'shell_command': ['/bin/bash']} From c2fdab0e0291be2bebe9c041d0889cbb226b377e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 12 Apr 2023 12:50:47 -0700 Subject: [PATCH 23/86] Add a variety of basic data science packages into the image --- workbench-for-google-cloud-workstations/Dockerfile | 12 +++++++++++- .../deps/install_r_packages.sh | 10 ++++++++++ .../deps/py_packages.txt | 4 ++++ .../deps/r_packages.txt | 5 +++++ 4 files changed, 30 insertions(+), 1 deletion(-) create mode 100755 workbench-for-google-cloud-workstations/deps/install_r_packages.sh create mode 100644 workbench-for-google-cloud-workstations/deps/py_packages.txt create mode 100644 workbench-for-google-cloud-workstations/deps/r_packages.txt diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index c28b0004..5a8acb15 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -80,8 +80,17 @@ RUN curl -O https://cdn.rstudio.com/python/ubuntu-2004/pkgs/python-${PYTHON_VERS && rm -rf python-${PYTHON_VERSION_ALT}_1_amd64.deb \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \ + && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade pip \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade pip + +### Install basic data science packages for Python and R ### +COPY deps/* / +RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /py_packages.txt \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install -r /py_packages.txt \ + && ./install_r_packages.sh \ + && rm install_r_packages.sh py_packages.txt r_packages.txt ### Locale configuration ### RUN localedef -i en_US -f UTF-8 en_US.UTF-8 @@ -133,6 +142,7 @@ RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyt && /opt/python/jupyter/bin/jupyter-nbextension install --sys-prefix --py rsconnect_jupyter \ && /opt/python/jupyter/bin/jupyter-nbextension enable --sys-prefix --py rsconnect_jupyter \ && /opt/python/jupyter/bin/jupyter-serverextension enable --sys-prefix --py rsconnect_jupyter \ + && /opt/python/jupyter/bin/python -m ipykernel install --name py${PYTHON_VERSION} --display-name "Python ${PYTHON_VERSION}" \ && /opt/python/jupyter/bin/python -m ipykernel install --name py${PYTHON_VERSION_ALT} --display-name "Python ${PYTHON_VERSION_ALT}" RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ diff --git a/workbench-for-google-cloud-workstations/deps/install_r_packages.sh b/workbench-for-google-cloud-workstations/deps/install_r_packages.sh new file mode 100755 index 00000000..b836957a --- /dev/null +++ b/workbench-for-google-cloud-workstations/deps/install_r_packages.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +UBUNTU_CODENAME=$(lsb_release -cs) +CRAN_REPO="https://packagemanager.rstudio.com/cran/__linux__/${UBUNTU_CODENAME}/latest" + +r_packages=$(awk '{print "\"" $0 "\""}' r_packages.txt | paste -d',' -s -) +/opt/R/${R_VERSION}/bin/R --slave -e "install.packages(c(${r_packages}), repos = \"${CRAN_REPO}\")" +/opt/R/${R_VERSION_ALT}/bin/R --slave -e "install.packages(c(${r_packages}), repos = \"${CRAN_REPO}\")" diff --git a/workbench-for-google-cloud-workstations/deps/py_packages.txt b/workbench-for-google-cloud-workstations/deps/py_packages.txt new file mode 100644 index 00000000..cf9726a1 --- /dev/null +++ b/workbench-for-google-cloud-workstations/deps/py_packages.txt @@ -0,0 +1,4 @@ +numpy +scipy +pandas +matplotlib \ No newline at end of file diff --git a/workbench-for-google-cloud-workstations/deps/r_packages.txt b/workbench-for-google-cloud-workstations/deps/r_packages.txt new file mode 100644 index 00000000..ca3e3219 --- /dev/null +++ b/workbench-for-google-cloud-workstations/deps/r_packages.txt @@ -0,0 +1,5 @@ +rmarkdown +shiny +tidyverse +tidymodels +odbc \ No newline at end of file From 8dd907ad2732499bbd05be4fa83e0460ec5ef304 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 13 Apr 2023 11:55:12 -0700 Subject: [PATCH 24/86] Add lab-args --- workbench-for-google-cloud-workstations/conf/jupyter.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/conf/jupyter.conf b/workbench-for-google-cloud-workstations/conf/jupyter.conf index 1727fbc4..0569e666 100644 --- a/workbench-for-google-cloud-workstations/conf/jupyter.conf +++ b/workbench-for-google-cloud-workstations/conf/jupyter.conf @@ -1,6 +1,8 @@ notebooks-enabled=1 labs-enabled=1 -jupyter-exe=/usr/local/bin/jupyter +jupyter-exe=/opt/python/jupyter/bin/jupyter + +lab-args=--no-browser --allow-root --ip=0.0.0.0 --allow-origin="*" default-session-cluster=Local From beda951b2ff69989142d6e893f9368d6318ac692 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 13 Apr 2023 13:14:10 -0700 Subject: [PATCH 25/86] Fix allow_origin arg --- workbench-for-google-cloud-workstations/conf/jupyter.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workbench-for-google-cloud-workstations/conf/jupyter.conf b/workbench-for-google-cloud-workstations/conf/jupyter.conf index 0569e666..25d9fa8a 100644 --- a/workbench-for-google-cloud-workstations/conf/jupyter.conf +++ b/workbench-for-google-cloud-workstations/conf/jupyter.conf @@ -1,8 +1,7 @@ notebooks-enabled=1 labs-enabled=1 -jupyter-exe=/opt/python/jupyter/bin/jupyter - -lab-args=--no-browser --allow-root --ip=0.0.0.0 --allow-origin="*" +jupyter-exe=/usr/local/bin/jupyter +lab-args="--ServerApp.allow_origin=\"*\"" default-session-cluster=Local From b6b5f413f35fda08b4d9fefc2f5ccdbbcab77bff Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 13 Apr 2023 14:18:14 -0700 Subject: [PATCH 26/86] Revert jupyter.conf changes --- workbench-for-google-cloud-workstations/conf/jupyter.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/conf/jupyter.conf b/workbench-for-google-cloud-workstations/conf/jupyter.conf index 25d9fa8a..1727fbc4 100644 --- a/workbench-for-google-cloud-workstations/conf/jupyter.conf +++ b/workbench-for-google-cloud-workstations/conf/jupyter.conf @@ -2,6 +2,5 @@ notebooks-enabled=1 labs-enabled=1 jupyter-exe=/usr/local/bin/jupyter -lab-args="--ServerApp.allow_origin=\"*\"" default-session-cluster=Local From a9e4c67689feea4a16473ec1e664c3b8228a56bb Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 14 Apr 2023 12:13:50 -0700 Subject: [PATCH 27/86] Add Quarto and temporarily add args to Jupyter Lab for debugging --- workbench-for-google-cloud-workstations/Dockerfile | 9 ++++++++- .../conf/jupyter.conf | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 5a8acb15..8dd9215e 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -7,7 +7,7 @@ ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.10.10 ARG PYTHON_VERSION_ALT=3.9.16 ARG PYTHON_VERSION_JUPYTER=3.10.10 -ARG QUARTO_VERSION=1.2.335 +ARG QUARTO_VERSION=1.3.324 ARG DRIVERS_VERSION=2022.11.0 ARG RSW_VERSION=2023.03.0+386.pro1 ARG RSW_NAME=rstudio-workbench @@ -98,6 +98,12 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 +### Install Quarto ### +RUN curl -o quarto-linux-amd64.deb -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb \ + && apt-get install -yq ./quarto-linux-amd64.deb \ + && rm quarto-linux-amd64.deb + + ### Install Pro Drivers ### RUN apt-get update \ && apt-get install -yq --no-install-recommends unixodbc unixodbc-dev \ @@ -128,6 +134,7 @@ COPY startup-user-provisioning/* /startup/user-provisioning/ COPY startup/* /startup/base/ COPY supervisord.conf /etc/supervisor/supervisord.conf +### Install Jupyter and extensions ### RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyter \ && /opt/python/jupyter/bin/pip install \ jupyter \ diff --git a/workbench-for-google-cloud-workstations/conf/jupyter.conf b/workbench-for-google-cloud-workstations/conf/jupyter.conf index 1727fbc4..4fa88b69 100644 --- a/workbench-for-google-cloud-workstations/conf/jupyter.conf +++ b/workbench-for-google-cloud-workstations/conf/jupyter.conf @@ -2,5 +2,6 @@ notebooks-enabled=1 labs-enabled=1 jupyter-exe=/usr/local/bin/jupyter +lab-args=--no-browser --allow-root --ip=0.0.0.0 --ServerApp.allow_origin="*" --ServerApp.allow_remote_access=True --debug default-session-cluster=Local From 0bd44ff5e5ab87a22a2cded4ed4e81bb275809e4 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Apr 2023 09:18:40 -0700 Subject: [PATCH 28/86] Fix goss tests --- .../workflows/build-workbench-for-gcw.yaml | 20 +++++++++---------- .../Dockerfile | 10 ++++++++++ .../Justfile | 13 +++++++----- .../conf/jupyter.conf | 2 +- .../docker-compose.test.yml | 2 +- .../test/goss.yaml | 10 +++++----- 6 files changed, 35 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 6c2d92c2..75b34ee4 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -72,13 +72,13 @@ jobs: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.TAG_SAFE_VERSION }} -# - name: Test Image -# env: -# IMAGE_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest -# RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} -# R_VERSION: ${{ env.R_VERSION }} -# R_VERSION_ALT: ${{ env.R_VERSION_ALT }} -# PYTHON_VERSION: ${{ env.PYTHON_VERSION }} -# PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} -# run: -# docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut + - name: Test Image + env: + IMAGE_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest + RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} + R_VERSION: ${{ env.R_VERSION }} + R_VERSION_ALT: ${{ env.R_VERSION_ALT }} + PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} + run: + docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 8dd9215e..acd2d503 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -128,6 +128,16 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/rstudio-server/r-versions +# TODO: Remove this block and use S3 dev build +#COPY rstudio-workbench-2023.04.999-dev-999.pro9.deb /rstudio-workbench.deb +#RUN apt-get update \ +# && apt-get install -y --no-install-recommends ./rstudio-workbench.deb \ +# && rm ./rstudio-workbench.deb \ +# && apt-get autoremove -y \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/* \ +# && rm -rf /var/lib/rstudio-server/r-versions + COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh COPY startup-launcher/* /startup/launcher/ COPY startup-user-provisioning/* /startup/user-provisioning/ diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index de8f49d6..2e91406b 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -6,7 +6,7 @@ IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu1804" -RSW_VERSION := "2023.03.0+386.pro1" +RSW_VERSION := "2023.04.999-dev+999.pro9" RSW_LICENSE := "" RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" @@ -16,6 +16,7 @@ R_VERSION_ALT := "4.1.3" PYTHON_VERSION := "3.10.10" PYTHON_VERSION_ALT := "3.9.16" +PYTHON_VERSION_JUPYTER := "3.10.10" _make-default-tag: echo "{{IMAGE_PREFIX}}{{PRODUCT}}:$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" @@ -48,26 +49,28 @@ build VERSION=RSW_VERSION *TAGS="": --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ + --build-arg PYTHON_VERSION_JUPYTER="{{ PYTHON_VERSION_JUPYTER }}" \ --build-arg RSW_DOWNLOAD_URL="{{ RSW_DOWNLOAD_URL }}" \ --file=./Dockerfile . # Test Workbench image - just test rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 -test TAG=`just _make-default-tag` VERSION=RSW_VERSION CMD="": +test TAG=`just _make-default-tag` CMD="": #!/usr/bin/env bash set -euxo pipefail IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ VERSION }}" \ + RSW_VERSION="{{ RSW_VERSION }}" \ RSW_LICENSE="{{ RSW_LICENSE }}" \ RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ R_VERSION="{{ R_VERSION }}" \ R_VERSION_ALT="{{ R_VERSION_ALT }}" \ PYTHON_VERSION="{{ PYTHON_VERSION }}" \ PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ + PYTHON_VERSION_JUPYTER="{{ PYTHON_VERSION_JUPYTER }}" \ docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} # Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 -test-i TAG=`just _make-default-tag` VERSION=RSW_VERSION: - just test {{ TAG }} {{ VERSION }} bash +test-i TAG=`just _make-default-tag`: + just test {{ TAG }} bash # Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu1804-2022.07.2-576.pro12 run TAG=`just _make-default-tag` CMD="": diff --git a/workbench-for-google-cloud-workstations/conf/jupyter.conf b/workbench-for-google-cloud-workstations/conf/jupyter.conf index 4fa88b69..c38488e4 100644 --- a/workbench-for-google-cloud-workstations/conf/jupyter.conf +++ b/workbench-for-google-cloud-workstations/conf/jupyter.conf @@ -2,6 +2,6 @@ notebooks-enabled=1 labs-enabled=1 jupyter-exe=/usr/local/bin/jupyter -lab-args=--no-browser --allow-root --ip=0.0.0.0 --ServerApp.allow_origin="*" --ServerApp.allow_remote_access=True --debug +lab-args=--no-browser --allow-root --ip=0.0.0.0 --ServerApp.allow_origin="*" --ServerApp.allow_remote_access=True --LabApp.token="" --debug default-session-cluster=Local diff --git a/workbench-for-google-cloud-workstations/docker-compose.test.yml b/workbench-for-google-cloud-workstations/docker-compose.test.yml index b28c6f5f..db54a879 100644 --- a/workbench-for-google-cloud-workstations/docker-compose.test.yml +++ b/workbench-for-google-cloud-workstations/docker-compose.test.yml @@ -11,7 +11,7 @@ services: - R_VERSION - PYTHON_VERSION - PYTHON_VERSION_ALT - - JUPYTER_PYTHON_VERSION + - PYTHON_VERSION_JUPYTER - R_VERSION_ALT - RSW_LICENSE - RSW_LICENSE_SERVER diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index f6c6b160..a9ec06e7 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -2,12 +2,12 @@ user: rstudio-server: exists: true uid: 999 - gid: 999 + gid: 998 group: rstudio-server: exists: true - gid: 999 + gid: 998 package: rstudio-server: @@ -106,14 +106,14 @@ command: title: jupyter_works exit-status: 0 # Ensure correct python version - "/opt/python/3.8.10/bin/python --version": + "/opt/python/3.9.16/bin/python --version": title: alternate_python_version_matches exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION_ALT}}" ] # Ensure correct python version - "/opt/python/3.9.5/bin/python --version": + "/opt/python/3.10.10/bin/python --version": title: python_version_matches exit-status: 0 stdout: [ @@ -123,7 +123,7 @@ command: title: jupyter_python_version_matches exit-status: 0 stdout: [ - "{{.Env.JUPYTER_PYTHON_VERSION}}" + "{{.Env.PYTHON_VERSION_JUPYTER}}" ] "which openssl": title: uses_system_openssl From 415f76d35e9facb529502c496fd504416dde50c6 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Apr 2023 09:58:24 -0700 Subject: [PATCH 29/86] Add PYTHON_VERSION_JUPYTER in workflow --- .github/workflows/build-workbench-for-gcw.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 75b34ee4..e9559c36 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -68,6 +68,7 @@ jobs: R_VERSION_ALT=${{ env.R_VERSION_ALT }} PYTHON_VERSION=${{ env.PYTHON_VERSION }} PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} + PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} tags: | us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.TAG_SAFE_VERSION }} @@ -80,5 +81,6 @@ jobs: R_VERSION_ALT: ${{ env.R_VERSION_ALT }} PYTHON_VERSION: ${{ env.PYTHON_VERSION }} PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} + PYTHON_VERSION_JUPYTER: ${{ env.PYTHON_VERSION }} run: docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut From ad6bec74a893a3f1debfe6ccd381c0aa5630b639 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Apr 2023 12:32:22 -0700 Subject: [PATCH 30/86] Fixes to build from daily with JupyterLab fix --- .../workflows/build-workbench-for-gcw.yaml | 21 +++++++++++-------- .../Dockerfile | 2 +- .../Justfile | 7 ++++--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index e9559c36..176876f8 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -14,6 +14,8 @@ jobs: R_VERSION_ALT: 4.1.3 PYTHON_VERSION: 3.10.10 PYTHON_VERSION_ALT: 3.9.16 + RSW_VERSION: 2023.05.0-daily-271.pro3 + RSW_DOWNLOAD_URL: https://s3.amazonaws.com/rstudio-ide-build/server/bionic/amd64 steps: - name: Check Out Repo @@ -43,13 +45,13 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- - - name: Get Version - id: get-version - run: | - VERSION=`just -f ci.Justfile get-version workbench --type=release --local` - VERSION=${VERSION//\"/} - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - echo "TAG_SAFE_VERSION=${VERSION/+/-}" >> $GITHUB_OUTPUT +# - name: Get Version +# id: get-version +# run: | +# VERSION=`just -f ci.Justfile get-version workbench --type=release --local` +# VERSION=${VERSION//\"/} +# echo "VERSION=$VERSION" >> $GITHUB_OUTPUT +# echo "TAG_SAFE_VERSION=${VERSION/+/-}" >> $GITHUB_OUTPUT - name: Authenticate gcloud run: | @@ -63,7 +65,8 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache build-args: | - RSW_VERSION=${{ steps.get-version.outputs.VERSION }} + RSW_VERSION=${{ env.RSW_VERSION }} + RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} R_VERSION=${{ env.R_VERSION }} R_VERSION_ALT=${{ env.R_VERSION_ALT }} PYTHON_VERSION=${{ env.PYTHON_VERSION }} @@ -76,7 +79,7 @@ jobs: - name: Test Image env: IMAGE_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest - RSW_VERSION: ${{ steps.get-version.outputs.VERSION }} + RSW_VERSION: ${{ env.RSW_VERSION }} R_VERSION: ${{ env.R_VERSION }} R_VERSION_ALT: ${{ env.R_VERSION_ALT }} PYTHON_VERSION: ${{ env.PYTHON_VERSION }} diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index acd2d503..2d0572e2 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -128,7 +128,7 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/rstudio-server/r-versions -# TODO: Remove this block and use S3 dev build +# TODO: Remove this block at initial release, but for now it may be helpful for testing any Workbench fixes #COPY rstudio-workbench-2023.04.999-dev-999.pro9.deb /rstudio-workbench.deb #RUN apt-get update \ # && apt-get install -y --no-install-recommends ./rstudio-workbench.deb \ diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 2e91406b..49b20c72 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -6,7 +6,7 @@ IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu1804" -RSW_VERSION := "2023.04.999-dev+999.pro9" +RSW_VERSION := "2023.05.0-daily-271.pro3" RSW_LICENSE := "" RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" @@ -31,7 +31,7 @@ build VERSION=RSW_VERSION *TAGS="": fi if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just _make-default-tag)) + raw_tag_array=($(just IMAGE_PREFIX="{{IMAGE_PREFIX}}" RSW_VERSION="{{RSW_VERSION}}" _make-default-tag)) else raw_tag_array=("{{TAGS}}") fi @@ -57,8 +57,9 @@ build VERSION=RSW_VERSION *TAGS="": test TAG=`just _make-default-tag` CMD="": #!/usr/bin/env bash set -euxo pipefail + RSW_VERSION_CLEAN=$(sed "s/daily-/daily+/" <<<{{RSW_VERSION}}) IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="{{ RSW_VERSION }}" \ + RSW_VERSION="$RSW_VERSION_CLEAN" \ RSW_LICENSE="{{ RSW_LICENSE }}" \ RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ R_VERSION="{{ R_VERSION }}" \ From 9139881adcb50e597096a6cac1d836de892e600b Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Apr 2023 12:57:28 -0700 Subject: [PATCH 31/86] Replace tagged version variable --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 176876f8..01b38f47 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -74,7 +74,7 @@ jobs: PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} tags: | us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ steps.get-version.outputs.TAG_SAFE_VERSION }} + us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ env.RSW_VERSION }} - name: Test Image env: From b39e39d4b19186bffb557e69f0882dddb37bfa8b Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Apr 2023 16:23:07 -0700 Subject: [PATCH 32/86] Update r_packages.txt with R Package Dependencies list --- .../deps/r_packages.txt | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/deps/r_packages.txt b/workbench-for-google-cloud-workstations/deps/r_packages.txt index ca3e3219..401fa223 100644 --- a/workbench-for-google-cloud-workstations/deps/r_packages.txt +++ b/workbench-for-google-cloud-workstations/deps/r_packages.txt @@ -1,5 +1,67 @@ +DBI +R6 +RJDBC +RODBC +RSQLite +Rcpp +base64enc +checkmate +crayon +curl +devtools +digest +ellipsis +evaluate +fastmap +glue +haven +highr +htmltools +htmlwidgets +httpuv +jsonlite +keyring +knitr +later +learnr +magrittr +markdown +mime +miniUI +mongolite +odbc +openssl +packrat +plumber +png +profvis +promises +r2d3 +ragg +rappdirs +rJava +readr +readxl +renv +reticulate +rlang rmarkdown +roxygen2 +rprojroot +rsconnect +rstan +rstudioapi shiny +shinytest +sourcetools +stringi +stringr +testthat tidyverse tidymodels -odbc \ No newline at end of file +tinytex +withr +xfun +xml2 +xtable +yaml From 76f2157f93630a0b51ac48333dbd78117f896615 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Apr 2023 08:04:05 -0700 Subject: [PATCH 33/86] Update test and push blocks in workflow --- .../workflows/build-workbench-for-gcw.yaml | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 01b38f47..be40023b 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -53,14 +53,14 @@ jobs: # echo "VERSION=$VERSION" >> $GITHUB_OUTPUT # echo "TAG_SAFE_VERSION=${VERSION/+/-}" >> $GITHUB_OUTPUT - - name: Authenticate gcloud + - name: Authenticate GCAR run: | gcloud auth configure-docker -q us-central1-docker.pkg.dev - name: Build Image uses: docker/build-push-action@v4 with: - push: true + push: false context: ./workbench-for-google-cloud-workstations cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache @@ -85,5 +85,31 @@ jobs: PYTHON_VERSION: ${{ env.PYTHON_VERSION }} PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} PYTHON_VERSION_JUPYTER: ${{ env.PYTHON_VERSION }} - run: - docker-compose -f ./workbench-for-google-cloud-workstations/docker-compose.test.yml run sut + run: | + just \ + RSW_VERSION=${{ env.RSW_VERSION }} \ + R_VERSION=${{ env.R_VERSION }} \ + R_VERSION_ALT=${{ env.R_VERSION_ALT }} \ + PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ + PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ + PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ + test + + - name: Push Image + uses: docker/build-push-action@v4 + with: + push: true + context: ./workbench-for-google-cloud-workstations + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + build-args: | + RSW_VERSION=${{ env.RSW_VERSION }} + RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} + R_VERSION=${{ env.R_VERSION }} + R_VERSION_ALT=${{ env.R_VERSION_ALT }} + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} + PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} + tags: | + us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, + us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ env.RSW_VERSION }} From c352f56620a740e263fa98168566b343dcf58ea8 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Apr 2023 09:20:13 -0700 Subject: [PATCH 34/86] Fix test path --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index be40023b..642b4533 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -93,7 +93,7 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ - test + workbench-for-google-cloud-workstations/test - name: Push Image uses: docker/build-push-action@v4 From 0ece256b1ec04cadbc18879e843de6649e28fe47 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Apr 2023 09:43:58 -0700 Subject: [PATCH 35/86] Specify tag for test --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 642b4533..5669941a 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -93,7 +93,7 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ - workbench-for-google-cloud-workstations/test + workbench-for-google-cloud-workstations/test us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest - name: Push Image uses: docker/build-push-action@v4 From 2216c8e3c5a71b5fede6e6fc7a760fb7144398c6 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Apr 2023 12:53:07 -0700 Subject: [PATCH 36/86] Use just for build pipeline --- .../workflows/build-workbench-for-gcw.yaml | 58 ++++--------------- .../Justfile | 6 +- 2 files changed, 15 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 5669941a..2c2ea95f 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -53,39 +53,15 @@ jobs: # echo "VERSION=$VERSION" >> $GITHUB_OUTPUT # echo "TAG_SAFE_VERSION=${VERSION/+/-}" >> $GITHUB_OUTPUT - - name: Authenticate GCAR + - name: Build base image + id: build run: | - gcloud auth configure-docker -q us-central1-docker.pkg.dev - - - name: Build Image - uses: docker/build-push-action@v4 - with: - push: false - context: ./workbench-for-google-cloud-workstations - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - build-args: | - RSW_VERSION=${{ env.RSW_VERSION }} - RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} - R_VERSION=${{ env.R_VERSION }} - R_VERSION_ALT=${{ env.R_VERSION_ALT }} - PYTHON_VERSION=${{ env.PYTHON_VERSION }} - PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} - PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} - tags: | - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ env.RSW_VERSION }} + TAGS=`just BUILDX_PATH=${{ steps.buildx.outputs.name }} RSW_VERSION=${{ env.RSW_VERSION }} RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} R_VERSION=${{ env.R_VERSION }} R_VERSION_ALT=${{ env.R_VERSION_ALT }} PYTHON_VERSION=${{ env.PYTHON_VERSION }} PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} workbench-for-google-cloud-workstations/build` + echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - name: Test Image - env: - IMAGE_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest - RSW_VERSION: ${{ env.RSW_VERSION }} - R_VERSION: ${{ env.R_VERSION }} - R_VERSION_ALT: ${{ env.R_VERSION_ALT }} - PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - PYTHON_VERSION_ALT: ${{ env.PYTHON_VERSION_ALT }} - PYTHON_VERSION_JUPYTER: ${{ env.PYTHON_VERSION }} run: | + IFS=" " read -a tags <<<${{ steps.build.outputs.TAGS }} just \ RSW_VERSION=${{ env.RSW_VERSION }} \ R_VERSION=${{ env.R_VERSION }} \ @@ -93,23 +69,11 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ - workbench-for-google-cloud-workstations/test us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest + workbench-for-google-cloud-workstations/test "${tags[1]}" + + - name: Authenticate GCAR + run: | + gcloud auth configure-docker -q us-central1-docker.pkg.dev - name: Push Image - uses: docker/build-push-action@v4 - with: - push: true - context: ./workbench-for-google-cloud-workstations - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - build-args: | - RSW_VERSION=${{ env.RSW_VERSION }} - RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} - R_VERSION=${{ env.R_VERSION }} - R_VERSION_ALT=${{ env.R_VERSION_ALT }} - PYTHON_VERSION=${{ env.PYTHON_VERSION }} - PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} - PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} - tags: | - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest, - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ env.RSW_VERSION }} + run: docker push ${{ steps.build-base-image.outputs.TAGS }} diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 49b20c72..ab96dfa4 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -22,7 +22,7 @@ _make-default-tag: echo "{{IMAGE_PREFIX}}{{PRODUCT}}:$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" # Build Workbench image - just build 2022.07.2+576.pro12 rstudio/rstudio-workbench-gcw:2022.07.2 -build VERSION=RSW_VERSION *TAGS="": +build *TAGS="": #!/usr/bin/env bash set -euxo pipefail BUILDX_ARGS="" @@ -44,7 +44,7 @@ build VERSION=RSW_VERSION *TAGS="": docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ VERSION }}" \ + --build-arg RSW_VERSION="{{ RSW_VERSION }}" \ --build-arg R_VERSION="{{ R_VERSION }}" \ --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ @@ -53,6 +53,8 @@ build VERSION=RSW_VERSION *TAGS="": --build-arg RSW_DOWNLOAD_URL="{{ RSW_DOWNLOAD_URL }}" \ --file=./Dockerfile . + echo ${raw_tag_array[@]} + # Test Workbench image - just test rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 test TAG=`just _make-default-tag` CMD="": #!/usr/bin/env bash From 56636e0ac3e727ef33f7f0d45d184bd08c01bb4a Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Apr 2023 14:08:19 -0700 Subject: [PATCH 37/86] Add tags to build step --- .github/workflows/build-workbench-for-gcw.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 2c2ea95f..442540c2 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -56,7 +56,18 @@ jobs: - name: Build base image id: build run: | - TAGS=`just BUILDX_PATH=${{ steps.buildx.outputs.name }} RSW_VERSION=${{ env.RSW_VERSION }} RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} R_VERSION=${{ env.R_VERSION }} R_VERSION_ALT=${{ env.R_VERSION_ALT }} PYTHON_VERSION=${{ env.PYTHON_VERSION }} PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} workbench-for-google-cloud-workstations/build` + TAGS=$(just \ + BUILDX_PATH=${{ steps.buildx.outputs.name }} \ + RSW_VERSION=${{ env.RSW_VERSION }} \ + RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} \ + R_VERSION=${{ env.R_VERSION }} \ + R_VERSION_ALT=${{ env.R_VERSION_ALT }} \ + PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ + PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ + PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ + workbench-for-google-cloud-workstations/build \ + us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest \ + us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ env.RSW_VERSION }}) echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - name: Test Image From c6826c837659e76dfd90dcfd0a72249574c6f382 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Apr 2023 14:33:18 -0700 Subject: [PATCH 38/86] Quote tags --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 442540c2..b599ed2a 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -72,7 +72,7 @@ jobs: - name: Test Image run: | - IFS=" " read -a tags <<<${{ steps.build.outputs.TAGS }} + IFS=" " read -a tags <<<"${{ steps.build.outputs.TAGS }}" just \ RSW_VERSION=${{ env.RSW_VERSION }} \ R_VERSION=${{ env.R_VERSION }} \ From 2b143d9d8624b5dd9883ce690dee63f7ec3cfeb0 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Apr 2023 15:00:26 -0700 Subject: [PATCH 39/86] Last fix for pushing tags --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index b599ed2a..00e2aae6 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -87,4 +87,4 @@ jobs: gcloud auth configure-docker -q us-central1-docker.pkg.dev - name: Push Image - run: docker push ${{ steps.build-base-image.outputs.TAGS }} + run: docker push ${{ steps.build.outputs.TAGS }} From feacba9db262c7969a0d9641b640518cb16ddb98 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Apr 2023 16:08:11 -0700 Subject: [PATCH 40/86] Use just command for pushing --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 00e2aae6..ed121963 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -87,4 +87,4 @@ jobs: gcloud auth configure-docker -q us-central1-docker.pkg.dev - name: Push Image - run: docker push ${{ steps.build.outputs.TAGS }} + run: just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} From 5892faad71dc0d92deecc851abdd9c704e3bb59d Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Apr 2023 08:19:30 -0700 Subject: [PATCH 41/86] Fixes after push failure --- .../workflows/build-workbench-for-gcw.yaml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index ed121963..b851052b 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -23,11 +23,6 @@ jobs: with: ref: gcw-workbench - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v1 - with: - credentials_json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - name: Set up Just uses: extractions/setup-just@v1 env: @@ -37,6 +32,15 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v1 + with: + credentials_json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + - name: Authenticate GCAR + run: | + gcloud auth configure-docker -q us-central1-docker.pkg.dev + - name: Cache Docker layers uses: actions/cache@v3 with: @@ -80,11 +84,7 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ - workbench-for-google-cloud-workstations/test "${tags[1]}" - - - name: Authenticate GCAR - run: | - gcloud auth configure-docker -q us-central1-docker.pkg.dev + workbench-for-google-cloud-workstations/test "${tags[0]}" - name: Push Image - run: just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} + run: docker push -a us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw From 9684acf3e66e122ce58b940451d574be280989cb Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Apr 2023 10:08:27 -0700 Subject: [PATCH 42/86] Implement cache inflation bug workaround --- .github/workflows/build-workbench-for-gcw.yaml | 12 +++++++++--- workbench-for-google-cloud-workstations/Justfile | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index b851052b..0e643c33 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -16,6 +16,7 @@ jobs: PYTHON_VERSION_ALT: 3.9.16 RSW_VERSION: 2023.05.0-daily-271.pro3 RSW_DOWNLOAD_URL: https://s3.amazonaws.com/rstudio-ide-build/server/bionic/amd64 + IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw steps: - name: Check Out Repo @@ -70,8 +71,8 @@ jobs: PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ workbench-for-google-cloud-workstations/build \ - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:latest \ - us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw:${{ env.RSW_VERSION }}) + ${{ env.IMAGE_REGISTRY_NAME }}:latest \ + ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_VERSION }}) echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - name: Test Image @@ -87,4 +88,9 @@ jobs: workbench-for-google-cloud-workstations/test "${tags[0]}" - name: Push Image - run: docker push -a us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw + run: docker push -a ${{ env.IMAGE_REGISTRY_NAME }} + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index ab96dfa4..ca179235 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -27,7 +27,7 @@ build *TAGS="": set -euxo pipefail BUILDX_ARGS="" if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" + BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max" fi if [[ "{{TAGS}}" == "" ]]; then From de07b3c890c64d1dc9bd97b8ed287d44a09b3d18 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Apr 2023 10:47:42 -0700 Subject: [PATCH 43/86] Push all tags --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 0e643c33..ee844cd9 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -88,7 +88,7 @@ jobs: workbench-for-google-cloud-workstations/test "${tags[0]}" - name: Push Image - run: docker push -a ${{ env.IMAGE_REGISTRY_NAME }} + run: just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} - name: Move cache run: | From 946a3f474e786977aed483f73e825197143d88d6 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Apr 2023 12:11:43 -0700 Subject: [PATCH 44/86] Run docker image ls to see why images aren't found --- .github/workflows/build-workbench-for-gcw.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index ee844cd9..c3ecf562 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -88,7 +88,9 @@ jobs: workbench-for-google-cloud-workstations/test "${tags[0]}" - name: Push Image - run: just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} + run: | + docker image ls + just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} - name: Move cache run: | From 24ec2e40293bb4839e4e9f3060404a23ba9d99af Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Apr 2023 13:59:31 -0700 Subject: [PATCH 45/86] Package and dependency updates --- workbench-for-google-cloud-workstations/Dockerfile | 4 ++++ workbench-for-google-cloud-workstations/Justfile | 2 +- workbench-for-google-cloud-workstations/conf/repos.conf | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 2d0572e2..0c190944 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -57,6 +57,10 @@ RUN apt-get update --fix-missing \ rrdtool \ sssd \ supervisor \ + tcl \ + tk \ + tk-dev \ + tk-table \ && rm -rf /var/lib/apt/lists/* ### Install R versions ### diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index ca179235..a0809fe6 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -37,7 +37,7 @@ build *TAGS="": fi tag_array=() - for tag in $raw_tag_array + for tag in ${raw_tag_array[@]}; do tag_array+=("-t" $tag) done diff --git a/workbench-for-google-cloud-workstations/conf/repos.conf b/workbench-for-google-cloud-workstations/conf/repos.conf index bee0ac41..d50b1707 100644 --- a/workbench-for-google-cloud-workstations/conf/repos.conf +++ b/workbench-for-google-cloud-workstations/conf/repos.conf @@ -1,2 +1,2 @@ -RSPM=https://packagemanager.rstudio.com/cran/__linux__/bionic/latest -CRAN=https://packagemanager.rstudio.com/cran/__linux__/bionic/latest +RSPM=https://packagemanager.rstudio.com/cran/__linux__/focal/latest +CRAN=https://packagemanager.rstudio.com/cran/__linux__/focal/latest From ece18d2c04ca79330454917f77f8d284a66f9c49 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Apr 2023 07:30:08 -0700 Subject: [PATCH 46/86] Just retag the image and forget about this problem for now --- .github/workflows/build-workbench-for-gcw.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index c3ecf562..15979621 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -71,8 +71,7 @@ jobs: PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ workbench-for-google-cloud-workstations/build \ - ${{ env.IMAGE_REGISTRY_NAME }}:latest \ - ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_VERSION }}) + ${{ env.IMAGE_REGISTRY_NAME }}:latest ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_VERSION }}) echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - name: Test Image @@ -86,10 +85,12 @@ jobs: PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ workbench-for-google-cloud-workstations/test "${tags[0]}" - + + # FIXME: The second tag isn't persisting for some reason, having to reapply it here - name: Push Image run: | docker image ls + docker tag ${{ steps.build.outputs.TAGS }} just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} - name: Move cache From fc0695c7cd4d202f40d92222af60856bb5626f03 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Apr 2023 15:07:00 -0700 Subject: [PATCH 47/86] Update registry name to release registry --- .github/workflows/build-workbench-for-gcw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 15979621..8886fd83 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -16,7 +16,7 @@ jobs: PYTHON_VERSION_ALT: 3.9.16 RSW_VERSION: 2023.05.0-daily-271.pro3 RSW_DOWNLOAD_URL: https://s3.amazonaws.com/rstudio-ide-build/server/bionic/amd64 - IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/gcp-workstations-testing/posit/workbench-for-gcw + IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench steps: - name: Check Out Repo From f23851ac251bfe3a783f3d554cfca9bc271cec4f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Apr 2023 17:04:12 -0700 Subject: [PATCH 48/86] Temporarily continue on error for tests --- .github/workflows/build-workbench-for-gcw.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 8886fd83..5a1138a6 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -75,6 +75,7 @@ jobs: echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - name: Test Image + continue-on-error: true run: | IFS=" " read -a tags <<<"${{ steps.build.outputs.TAGS }}" just \ From 02919f1f523eacd5bf187da6620155dce158c689 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 2 May 2023 12:54:54 -0700 Subject: [PATCH 49/86] Improve Goss tests based on #490 Add missing R package system library dependencies Change CRAN repos to use posit.co address Upgrade Goss to 0.3.22 --- .../Dockerfile | 2 + .../Justfile | 9 +- .../NEWS.md | 58 ------- .../conf/repos.conf | 4 +- .../deps/install_r_packages.sh | 2 +- .../docker-compose.test.yml | 8 +- .../test/goss.yaml | 150 +++++++++++++++--- .../test/run_tests.sh | 2 +- 8 files changed, 144 insertions(+), 91 deletions(-) delete mode 100644 workbench-for-google-cloud-workstations/NEWS.md diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 0c190944..91c4748b 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -34,6 +34,7 @@ RUN apt-get update --fix-missing \ apt-transport-https \ build-essential \ ca-certificates \ + default-jdk \ dirmngr \ dpkg-sig \ gnupg \ @@ -46,6 +47,7 @@ RUN apt-get update --fix-missing \ libpam-sss \ libpq-dev \ libpq5 \ + libsecret-1-dev \ libsm6 \ libssl-dev \ libuser \ diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index a0809fe6..cc80cf5a 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -4,12 +4,13 @@ BUILDX_PATH := "" IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" -IMAGE_OS := "ubuntu1804" +IMAGE_OS := "ubuntu2004" RSW_VERSION := "2023.05.0-daily-271.pro3" RSW_LICENSE := "" RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" +DRIVERS_VERSION := "2022.11.0" R_VERSION := "4.2.2" R_VERSION_ALT := "4.1.3" @@ -18,6 +19,8 @@ PYTHON_VERSION := "3.10.10" PYTHON_VERSION_ALT := "3.9.16" PYTHON_VERSION_JUPYTER := "3.10.10" +QUARTO_VERSION := "1.3.324" + _make-default-tag: echo "{{IMAGE_PREFIX}}{{PRODUCT}}:$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" @@ -50,6 +53,8 @@ build *TAGS="": --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ --build-arg PYTHON_VERSION_JUPYTER="{{ PYTHON_VERSION_JUPYTER }}" \ + --build-arg QUARTO_VERSION="{{ QUARTO_VERSION }}" \ + --build-arg DRIVERS_VERSION="{{ DRIVERS_VERSION }}" \ --build-arg RSW_DOWNLOAD_URL="{{ RSW_DOWNLOAD_URL }}" \ --file=./Dockerfile . @@ -64,11 +69,13 @@ test TAG=`just _make-default-tag` CMD="": RSW_VERSION="$RSW_VERSION_CLEAN" \ RSW_LICENSE="{{ RSW_LICENSE }}" \ RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ + DRIVERS_VERSION="{{ DRIVERS_VERSION }}" \ R_VERSION="{{ R_VERSION }}" \ R_VERSION_ALT="{{ R_VERSION_ALT }}" \ PYTHON_VERSION="{{ PYTHON_VERSION }}" \ PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ PYTHON_VERSION_JUPYTER="{{ PYTHON_VERSION_JUPYTER }}" \ + QUARTO_VERSION="{{ QUARTO_VERSION }}" \ docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} # Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 diff --git a/workbench-for-google-cloud-workstations/NEWS.md b/workbench-for-google-cloud-workstations/NEWS.md deleted file mode 100644 index 4afefe75..00000000 --- a/workbench-for-google-cloud-workstations/NEWS.md +++ /dev/null @@ -1,58 +0,0 @@ -# 2022-01-23 - -- Add documentation for license leak bug and possible workarounds/solutions. -- Add an option in the `Justfile` to persist license state files for use -across multiple restarts. - -# 2022.12.0 - -- Upgrade workbench to 2022.12.0 -- Use the new `WORKBENCH_JUPYTER_PATH` env var to configure jupyter location -- Remove `vscode.conf/exe` configuration in favor of the new, internal default `code-server` installation -- Refactor image to build _FROM_ the new `product-base-pro` image - -# 2022-07-28 - -- Added `conf/launcher.local.conf` with `unprivileged=1` flag so the Local Launcher Plugin starts in unprivileged mode. - -# 2022.07.0 - -- Add a `/usr/local/bin/jupyter` symlink -- Configure Workbench to preinstall R, Python, and Quarto extensions for users on session startup -- Add workbench_jupyterlab python package for the jupyterlab extension - -# 2022-04-07 - -- The Dockerfile now uses BuildKit features and must be built with - DOCKER_BUILDKIT=1. - -# 2021.09.0 - -- **BREAKING**: rename environment variables to use `RSW_` prefix instead of `RSP_` prefix - - i.e. `RSP_LICENSE` is now `RSW_LICENSE` -- **BREAKING**: Change execution model significantly - - Before, we had a single startup script `startup.sh` that would launch multiple services - - There was no service manager, so launcher going down could not be detected - - We are now using `supervisord` as a service manager within the container - - `supervisord` is configured to exit if any of its child processes exit - - `ENTRYPOINT` and `CMD` have changed accordingly - - A more thorough write-up is in `README.md` -- Improve logging behaviors to send logs to `stdout`/`stderr` instead of using `tail` -- Add `sssd` to the container for user provisioning. It starts by default with no directory configured. - - To utilize, mount your own `*.conf` files into `/etc/sssd/conf.d/` (ensure ownership by root, and 0600 permissions) - - Examples and further documentation is in `README.md` - -# 1.4.1717-3 - -- Add `code-server` / vscode installation and configuration -- Added R version 4.1.0. Image includes versions 4.1.0, 3.6.3 -- BREAKING: change image name to `rstudio/rstudio-workbench` -- BREAKING: change the location / version of python. Now includes: - - version 3.8.10 at /opt/python/jupyter to run jupyter - - version 3.8.10 at /opt/python/3.8.10 as a kernel for jupyter - - version 3.9.5 at /opt/python/3.9.5 as a kernel for jupyter - -# 1.4.1106-5 - -- Includes R version 3.6.3 at /opt/R/3.6.3 -- Includes Python version 3.6.5 at /opt/python/3.6.5 diff --git a/workbench-for-google-cloud-workstations/conf/repos.conf b/workbench-for-google-cloud-workstations/conf/repos.conf index d50b1707..de4785ea 100644 --- a/workbench-for-google-cloud-workstations/conf/repos.conf +++ b/workbench-for-google-cloud-workstations/conf/repos.conf @@ -1,2 +1,2 @@ -RSPM=https://packagemanager.rstudio.com/cran/__linux__/focal/latest -CRAN=https://packagemanager.rstudio.com/cran/__linux__/focal/latest +CRAN=https://packagemanager.posit.co/cran/__linux__/focal/latest +RSPM=https://packagemanager.posit.co/cran/__linux__/focal/latest diff --git a/workbench-for-google-cloud-workstations/deps/install_r_packages.sh b/workbench-for-google-cloud-workstations/deps/install_r_packages.sh index b836957a..885ee27f 100755 --- a/workbench-for-google-cloud-workstations/deps/install_r_packages.sh +++ b/workbench-for-google-cloud-workstations/deps/install_r_packages.sh @@ -3,7 +3,7 @@ set -ex UBUNTU_CODENAME=$(lsb_release -cs) -CRAN_REPO="https://packagemanager.rstudio.com/cran/__linux__/${UBUNTU_CODENAME}/latest" +CRAN_REPO="https://packagemanager.posit.co/cran/__linux__/${UBUNTU_CODENAME}/latest" r_packages=$(awk '{print "\"" $0 "\""}' r_packages.txt | paste -d',' -s -) /opt/R/${R_VERSION}/bin/R --slave -e "install.packages(c(${r_packages}), repos = \"${CRAN_REPO}\")" diff --git a/workbench-for-google-cloud-workstations/docker-compose.test.yml b/workbench-for-google-cloud-workstations/docker-compose.test.yml index db54a879..8de098c6 100644 --- a/workbench-for-google-cloud-workstations/docker-compose.test.yml +++ b/workbench-for-google-cloud-workstations/docker-compose.test.yml @@ -8,13 +8,17 @@ services: environment: # uses .env by default - RSW_VERSION - - R_VERSION + - R_VERSION + - R_VERSION_ALT - PYTHON_VERSION - PYTHON_VERSION_ALT - PYTHON_VERSION_JUPYTER - - R_VERSION_ALT + - QUARTO_VERSION + - DRIVERS_VERSION - RSW_LICENSE - RSW_LICENSE_SERVER volumes: - "./test/run_tests.sh:/run_tests.sh" - "./test/goss.yaml:/tmp/goss.yaml" + - "./deps/py_packages.txt:/tmp/py_packages.txt" + - "./deps/r_packages.txt:/tmp/r_packages.txt" diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index a9ec06e7..92c270cc 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -12,6 +12,24 @@ group: package: rstudio-server: installed: true + versions: + - {{ .Env.RSW_VERSION }} + r-{{.Env.R_VERSION}}: + installed: true + r-{{.Env.R_VERSION_ALT}}: + installed: true + python-{{.Env.PYTHON_VERSION}}: + installed: true + python-{{.Env.PYTHON_VERSION_ALT}}: + installed: true + quarto: + installed: true + versions: + - {{.Env.QUARTO_VERSION}} + rstudio-drivers: + installed: true + versions: + - {{.Env.DRIVERS_VERSION}} port: tcp:80: @@ -35,8 +53,6 @@ process: file: /etc/rstudio-server/license.lic: - # currently does not exist - # be sure it is NOT a directory exists: false /usr/lib/rstudio-server: exists: true @@ -56,9 +72,6 @@ file: exists: true /usr/local/bin/jupyter: exists: true - # this file gets created at startup? - # /var/lib/rstudio-server/r-versions: - # exists: false /usr/lib/rstudio-server/bin/rserver: exists: true /usr/lib/rstudio-server/bin/rstudio-server: @@ -73,6 +86,31 @@ file: exists: true /etc/rstudio/vscode.conf: exists: true + /etc/rstudio/jupyter.conf: + exists: true + contains: + - --LabApp.token="" + - --no-browser + - --allow-root + - --ip=0.0.0.0 + /etc/rstudio/rserver.conf: + exists: true + contains: + - www-port=80 + - launcher-sessions-callback-address=http://0.0.0.0:80 + - auth-proxy=1 + - auth-proxy-sign-in-url=http://localhost:80/ + - auth-proxy-user-header=x-custom-user-name + /etc/rstudio/nginx.site.conf: + exists: true + contains: + - proxy_set_header X-CUSTOM-USER-NAME user/google; + - proxy_set_header Host $http_x_forwarded_host; + /etc/rstudio/vscode.extensions.conf: + exists: true + contains: + - quarto.quarto + - GoogleCloudTools.cloudcode /tmp/startup.log: exists: true contains: @@ -86,48 +124,108 @@ file: owner: root group: root mode: "0600" + /usr/local/bin/wait-for-it.sh: + exists: true + owner: root + group: root + mode: "0755" + /etc/workstation-startup.d/010_add-user.sh: + exists: true + owner: root + group: root + mode: "0755" + filetype: file + contains: [ + "useradd -m user" + ] + /etc/workstation-startup.d/110_config-jupyter.sh: + exists: true + owner: root + group: root + mode: "0755" + filetype: file + contains: [ + # Checks that we're setting the Jupyter shell to /bin/bash + "echo \"c.ServerApp.terminado_settings = {'shell_command': ['/bin/bash']}\" > /home/user/.jupyter/jupyter_notebook_config.py" + ] + /etc/workstation-startup.d/120_start-workbench.sh: + exists: true + owner: root + group: root + mode: "0755" + filetype: file + contains: [ + "/usr/bin/supervisord -c /etc/supervisor/supervisord.conf" + ] + command: - "su rstudio-server -c 'touch /var/lib/rstudio-server/monitor/log/rstudio-server.log'": - title: workbench_monitor_log_file_creates + # Check OS release version (early heads up in case the base image is modified) + Check OS version is "focal": + exec: "lsb_release -cs" exit-status: 0 - "touch /var/log/rstudio-server.log": - title: workbench_server_log_creates + stdout: [ + "focal" + ] + Test Workbench Monitor rstudio-server.log creation: + exec: su rstudio-server -c 'touch /var/lib/rstudio-server/monitor/log/rstudio-server.log' + exit-status: 0 + Test rstudio-server.log creation: + exec: touch /var/log/rstudio-server.log exit-status: 0 -# Check product version - "rstudio-server version": - title: workbench_version_matches + Check RStudio Workbench version is {{ .Env.RSW_VERSION }}: + exec: rstudio-server version exit-status: 0 stdout: [ "{{ .Env.RSW_VERSION }}", "Workbench" ] - "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout": - title: jupyter_works + Test Jupyter works: + exec: "echo '{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2}' | /opt/python/jupyter/bin/jupyter nbconvert --to notebook --stdin --stdout" exit-status: 0 -# Ensure correct python version - "/opt/python/3.9.16/bin/python --version": - title: alternate_python_version_matches + Check primary Python is version {{.Env.PYTHON_VERSION}}: + exec: /opt/python/{{.Env.PYTHON_VERSION}}/bin/python --version exit-status: 0 stdout: [ - "{{.Env.PYTHON_VERSION_ALT}}" + "{{.Env.PYTHON_VERSION}}" ] -# Ensure correct python version - "/opt/python/3.10.10/bin/python --version": - title: python_version_matches + Check alternate Python is version {{.Env.PYTHON_VERSION_ALT}}: + exec: /opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python --version exit-status: 0 stdout: [ - "{{.Env.PYTHON_VERSION}}" + "{{.Env.PYTHON_VERSION_ALT}}" ] - "/opt/python/jupyter/bin/python --version": - title: jupyter_python_version_matches + Check Jupyter venv uses Python {{.Env.PYTHON_VERSION_JUPYTER}}: + exec: /opt/python/jupyter/bin/python --version exit-status: 0 stdout: [ "{{.Env.PYTHON_VERSION_JUPYTER}}" ] - "which openssl": - title: uses_system_openssl + Check for OpenSSL: + exec: which openssl exit-status: 0 stdout: [ "/usr/bin/openssl" ] + {{ $python_version := .Env.PYTHON_VERSION }} + {{ $python_version_alt := .Env.PYTHON_VERSION_ALT }} + {{ $py_package_list := readFile "/tmp/py_packages.txt" | splitList "\n" }} + {{- range $py_package_list }} + Check Python {{ $python_version }} has "{{.}}" installed: + exec: /opt/python/{{$python_version}}/bin/pip show {{.}} + exit-status: 0 + Check Python {{ $python_version_alt }} has "{{.}}" installed: + exec: /opt/python/{{$python_version_alt}}/bin/pip show {{.}} + exit-status: 0 + {{end}} + {{ $r_version := .Env.R_VERSION }} + {{ $r_version_alt := .Env.R_VERSION_ALT }} + {{ $r_package_list := readFile "/tmp/r_packages.txt" | splitList "\n" }} + {{- range $r_package_list }} + Check R {{ $r_version }} has "{{.}}" installed: + exec: /opt/R/{{$r_version}}/bin/R --slave -e "library(\"{{.}}\")" + exit-status: 0 + Check R {{ $r_version_alt }} has "{{.}}" installed: + exec: /opt/R/{{$r_version_alt}}/bin/R --slave -e "library(\"{{.}}\")" + exit-status: 0 + {{end}} \ No newline at end of file diff --git a/workbench-for-google-cloud-workstations/test/run_tests.sh b/workbench-for-google-cloud-workstations/test/run_tests.sh index c8d8e170..02c2cd04 100755 --- a/workbench-for-google-cloud-workstations/test/run_tests.sh +++ b/workbench-for-google-cloud-workstations/test/run_tests.sh @@ -17,7 +17,7 @@ echo "--> Startup complete" GOSS_FILE=${GOSS_FILE:-/tmp/goss.yaml} GOSS_VARS=${GOSS_VARS:-/tmp/goss_vars.yaml} -GOSS_VERSION=${GOSS_VERSION:-0.3.16} +GOSS_VERSION=${GOSS_VERSION:-0.3.22} GOSS_MAX_CONCURRENT=${GOSS_MAX_CONCURRENT:-50} # default to empty var file (since vars are not necessary) From 04a16fc923c50772f5047e7923666a3db31af675 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 3 May 2023 12:25:43 -0700 Subject: [PATCH 50/86] Update VSCode Extensions --- .../conf/vscode.extensions.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf b/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf index 9fe91992..15f24f1f 100644 --- a/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf +++ b/workbench-for-google-cloud-workstations/conf/vscode.extensions.conf @@ -1,4 +1,4 @@ quarto.quarto GoogleCloudTools.cloudcode -Ikuyadeu.r@2.4.0 -ms-python.python@2022.10.1 +REditorSupport.r +ms-python.python From 6602fbe17241e83979478c0fb8180f5b0488c828 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 8 May 2023 06:35:58 -0700 Subject: [PATCH 51/86] Disable pipeline caching --- .../workflows/build-workbench-for-gcw.yaml | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 5a1138a6..e30e8e31 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -42,13 +42,14 @@ jobs: run: | gcloud auth configure-docker -q us-central1-docker.pkg.dev - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- +# FIXME(ianpittwood): Caching keeps overrunning our disk space +# - name: Cache Docker layers +# uses: actions/cache@v3 +# with: +# path: /tmp/.buildx-cache +# key: ${{ runner.os }}-buildx-${{ github.sha }} +# restore-keys: | +# ${{ runner.os }}-buildx- # - name: Get Version # id: get-version @@ -94,7 +95,7 @@ jobs: docker tag ${{ steps.build.outputs.TAGS }} just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache +# - name: Move cache +# run: | +# rm -rf /tmp/.buildx-cache +# mv /tmp/.buildx-cache-new /tmp/.buildx-cache From 1326aa56629278be114cf6c9475e5bfa696737fb Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 16 May 2023 10:37:59 -0700 Subject: [PATCH 52/86] Disable pipeline caching --- workbench-for-google-cloud-workstations/Dockerfile | 12 +----------- workbench-for-google-cloud-workstations/Justfile | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 91c4748b..834cceee 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -9,7 +9,7 @@ ARG PYTHON_VERSION_ALT=3.9.16 ARG PYTHON_VERSION_JUPYTER=3.10.10 ARG QUARTO_VERSION=1.3.324 ARG DRIVERS_VERSION=2022.11.0 -ARG RSW_VERSION=2023.03.0+386.pro1 +ARG RSW_VERSION=2023.03.1+446.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 @@ -134,16 +134,6 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/rstudio-server/r-versions -# TODO: Remove this block at initial release, but for now it may be helpful for testing any Workbench fixes -#COPY rstudio-workbench-2023.04.999-dev-999.pro9.deb /rstudio-workbench.deb -#RUN apt-get update \ -# && apt-get install -y --no-install-recommends ./rstudio-workbench.deb \ -# && rm ./rstudio-workbench.deb \ -# && apt-get autoremove -y \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/* \ -# && rm -rf /var/lib/rstudio-server/r-versions - COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh COPY startup-launcher/* /startup/launcher/ COPY startup-user-provisioning/* /startup/user-provisioning/ diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index cc80cf5a..a8523b7e 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -6,7 +6,7 @@ IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu2004" -RSW_VERSION := "2023.05.0-daily-271.pro3" +RSW_VERSION := "2023.03.1+446.pro1" RSW_LICENSE := "" RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" From 94f0252e71f91865a43ea509437206c3dbb6579d Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 16 May 2023 11:16:25 -0700 Subject: [PATCH 53/86] Update RSW_VERSION var --- .github/workflows/build-workbench-for-gcw.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index e30e8e31..20c65cb2 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -14,8 +14,8 @@ jobs: R_VERSION_ALT: 4.1.3 PYTHON_VERSION: 3.10.10 PYTHON_VERSION_ALT: 3.9.16 - RSW_VERSION: 2023.05.0-daily-271.pro3 - RSW_DOWNLOAD_URL: https://s3.amazonaws.com/rstudio-ide-build/server/bionic/amd64 + RSW_VERSION: 2023.03.1-446.pro1 + #RSW_DOWNLOAD_URL: https://s3.amazonaws.com/rstudio-ide-build/server/bionic/amd64 IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench steps: From 4226145f3db40cdaac9e5842f43704885af55e7b Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 16 May 2023 13:03:59 -0700 Subject: [PATCH 54/86] Update launcher-env PATH --- workbench-for-google-cloud-workstations/conf/launcher-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/conf/launcher-env b/workbench-for-google-cloud-workstations/conf/launcher-env index a1144c18..6cbec885 100644 --- a/workbench-for-google-cloud-workstations/conf/launcher-env +++ b/workbench-for-google-cloud-workstations/conf/launcher-env @@ -3,4 +3,4 @@ Environment: LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 JobType: any -Environment: PATH=/opt/python/3.8.10/bin:$PATH +Environment: PATH=/opt/python/3.9.16/bin:$PATH From 68380231b08efa831b9d41230988a05d231ead6e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 19 May 2023 14:06:50 -0700 Subject: [PATCH 55/86] Upgrade R and Python versions Add patch to rserver.conf to fix libR.so not found bug --- .github/workflows/build-workbench-for-gcw.yaml | 6 ++++-- workbench-for-google-cloud-workstations/Dockerfile | 6 +++--- workbench-for-google-cloud-workstations/Justfile | 6 +++--- workbench-for-google-cloud-workstations/conf/rserver.conf | 1 + 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 20c65cb2..d14cc164 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -4,15 +4,17 @@ on: push: branches: - gcw-workbench + - main + - dev name: Build Workbench for Google Cloud Workstations jobs: build-workbench-for-gcw: runs-on: ubuntu-latest env: - R_VERSION: 4.2.2 + R_VERSION: 4.2.3 R_VERSION_ALT: 4.1.3 - PYTHON_VERSION: 3.10.10 + PYTHON_VERSION: 3.10.11 PYTHON_VERSION_ALT: 3.9.16 RSW_VERSION: 2023.03.1-446.pro1 #RSW_DOWNLOAD_URL: https://s3.amazonaws.com/rstudio-ide-build/server/bionic/amd64 diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 834cceee..5ef39f7e 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -2,11 +2,11 @@ FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive -ARG R_VERSION=4.2.2 +ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.10.10 +ARG PYTHON_VERSION=3.10.11 ARG PYTHON_VERSION_ALT=3.9.16 -ARG PYTHON_VERSION_JUPYTER=3.10.10 +ARG PYTHON_VERSION_JUPYTER=3.10.11 ARG QUARTO_VERSION=1.3.324 ARG DRIVERS_VERSION=2022.11.0 ARG RSW_VERSION=2023.03.1+446.pro1 diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index a8523b7e..57a30908 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -12,12 +12,12 @@ RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" DRIVERS_VERSION := "2022.11.0" -R_VERSION := "4.2.2" +R_VERSION := "4.2.3" R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.10.10" +PYTHON_VERSION := "3.10.11" PYTHON_VERSION_ALT := "3.9.16" -PYTHON_VERSION_JUPYTER := "3.10.10" +PYTHON_VERSION_JUPYTER := "3.10.11" QUARTO_VERSION := "1.3.324" diff --git a/workbench-for-google-cloud-workstations/conf/rserver.conf b/workbench-for-google-cloud-workstations/conf/rserver.conf index ddf0ba60..d9b416e4 100644 --- a/workbench-for-google-cloud-workstations/conf/rserver.conf +++ b/workbench-for-google-cloud-workstations/conf/rserver.conf @@ -4,6 +4,7 @@ admin-enabled=1 www-port=80 server-project-sharing=0 auth-pam-sessions-enabled=1 +rsession-no-profile=1 # Launcher Config launcher-address=127.0.0.1 From 36b1de8fddef0190ce48bb437b4dec55ac8f53cc Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 6 Jun 2023 09:56:49 -0700 Subject: [PATCH 56/86] License manager implementation for WGCW --- .../Dockerfile | 13 +++++++++++++ .../TurboActivate.dat | Bin 0 -> 4054 bytes .../conf/rserver.conf | 2 ++ .../license-manager-shim | 7 +++++++ .../startup.sh | 4 +++- 5 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 workbench-for-google-cloud-workstations/TurboActivate.dat create mode 100755 workbench-for-google-cloud-workstations/license-manager-shim diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 5ef39f7e..1c4b5ad7 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -27,6 +27,8 @@ ENV LICENSE_MANAGER_PATH /usr/lib/rstudio-server/bin ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter +COPY --chmod=755 TurboActivate.dat /opt/rstudio-license/license-manager.conf + ### Update/upgrade system packages ### RUN apt-get update --fix-missing \ && apt-get upgrade -yq \ @@ -134,6 +136,17 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/rstudio-server/r-versions +### Install GCW License Manager ### +# TODO(ianpittwood): Replace monitor download with $RSW_VERSION after upgrading to 2023.06.0 +RUN mkdir -p /opt/rstudio-license/ \ + && mkdir -p /var/lib/rstudio-workbench \ + && curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/focal/rsp-monitor-workbench-gcpw-amd64-2023.06.0-419.pro1.tar.gz" | \ + tar xzvf - --strip 2 -C /opt/rstudio-license/ \ + && chmod 0755 /opt/rstudio-license/license-manager \ + && mv /opt/rstudio-license/license-manager /opt/rstudio-license/license-manager-orig \ + && rm -f /usr/lib/rstudio-server/bin/license-manager + +COPY --chmod=755 license-manager-shim /opt/rstudio-license/license-manager COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh COPY startup-launcher/* /startup/launcher/ COPY startup-user-provisioning/* /startup/user-provisioning/ diff --git a/workbench-for-google-cloud-workstations/TurboActivate.dat b/workbench-for-google-cloud-workstations/TurboActivate.dat new file mode 100644 index 0000000000000000000000000000000000000000..d1b139fdafa4119532ded686ca9dfad365dcb29e GIT binary patch literal 4054 zcmV;{4=M0eK~O|SRxts;1ONaqf&{w)0RS)!1_>&LNQUrr7(g7p#lK_ z0)hbn0EXLHPcJJSCSGR+n+bHt!YIxF+9TWwCEM^_an$s?RQdEpe&^%D;iTuI#Fd(y zQr{P3iXX<|hs(^9^9rK0QVVQsFCKTE#%KT$S=X)meDI~z`WioHu6vx1lFv>OK5yX( zWGx3Re|#4#_UV+cDoSJqhTQQ1`&oH?6G0VGhefB-Z-5y$a|K%O3^WWZrPRP0?Rc0L+qBx!}ouVG&apre$6pZb1L65)g_sSLz?c} z4FUlX0)hbmH-~>9n)9NUR18g;F%kGr@~=e4o(OU~7T({fB4Gv>Po?VnxEu-CHZ@@aVc8^uIvOo5r8``DE# zJNd~07ie=U)JG2=8SqjmTA8Mc_y4Am@Z3FiVq}n#$_}4@X6ptaRFC-?>&BGjVyXb1 zcGql`NeEnf3L$7=nno;<@jxzS81y`2J|A`uClJg<;tpP>SO?wwyJhuSR0)#cb{K=m z41QNN0)c@5x$+@L`M?DruizoUiH^8(v1%s;B@!M|^-yaa2Y5WQ&T#X|*A@i?dqq#n z2P&XdOW9mmVq=P;}NK!O`FxHsBu++bh`tB36CUx#-AElc-Vo{vpyd%Ew(xZU^|9uy22kZiST z%BAtxPw=3#t?hyg)%BX8j%JW1#gz}+?LUQ2r$r8Gy1ig3T>e(WRsL#+yp7-ZzRg@t z6fl!BVHHwU^FkJ?*k!=7aYOwQ_ThuCxeWCiXbHof1UNJpK)U)xs0(99`qBrX*{o^O z0`pLt`M>f*0)c=`kl=?5d_GvfYMoG$6Kraa|XXAu1bo7~h%$?+9#s%b}c;cjUa=AW+Z1G~4=1cf86xb5gZvI(7IX3X09AeKhEJa%wrsqmyPQI1 zcbI;3?#XiMNTbwIpFh^?qJaVc3IG6P4F?YISR?1t zL6-r<_q-*|=NK8$R7yV%7pG#IciY1lf|`r=eY)=>CCM6+LJJZyO|V68c1yygp~^%u zMs!eaMT}gzmi?}`~LRln1s1E6eQiF`S~Y);5T%p5&tT!#<uzlNMVmhdqmiybMV0{f9oC-Dz&&ig>brrwOhFdxzt2}^N?5#j2 zr*>Ud+%o6s5+w`UC6xqMivQ}K-V1Cqj0~zkPlONvxpkuWr~*QN379}4<;B+D>PZo+l=jdhMSnSr%V4fMT}M%LrT z{Cs*Gwz%OnB7}=T>(#r{Xp9??t&vsUy$*SZpt*?Q8F$4_G=>C16`_b6vDkQuM}RSo7wlxnO}nNW@)I zsGgaw^Qr%Fqdz}PLhQF}HwfLek1m?oKV>1O$={N|07Gd;$z%LvP$k6`BFNmRN?zuV zbkG9rYuz}McQDZi7fRvmV>9^|L|eAdWDkpniptq6mPLHqQ!Vq3c};sG3VriK-H^G! zktm?M(BdY)J>C2#v=u=J4gJ&bLMZHqF69!cBjtRC;fIrfya5+Ym{J!N27}1agX$fS zzHwAGCKCrkmmu1icY6Trz1tKA9j-PMs^o{T;tLmZ-CImvjg$|=sMX_F`DaK-E>05M zpaGvJHRBbJ9xlM0MqV$NPg7Fq9#Nmw3AQ~L#-(vu03LnX?#*CX5~lUo9-+3v&EZKyj2w42Dxwu&D|&zJ9F3dcBr{;(p`fWz=RmX@mF zq=7BjyU%j|(U)fy@#4Rmv+=Iy`kZ0TAiD(?635lh@1)vFWaupJOXvsAF%K*JJl_3n zX%O62wbFc{;-eN@QtvqWMPM;yC!8Bu?Ib(r+z+%pmVmDn3V|o>%hz)1ZKGZnzXuD} zu-J^^x%~Er(X}lz9A#WlW}4j!e&e?XOOVzDvRXb-4D-{VT|T*nTuK#&8FO8Eb@G5V zksf&A5(f5(v9%#Bnu|aMw8zEg(^ZO!K9n#;FbEAB?+nTrT zAY7zK8I6fjzAH&gYK;Dy+*$WaS?Aar+LwiT!wPV$XT;ZtjX_FpjzE!7V+ui=&)2(i z$F^6CDePTo{38xBX+($iQhLd#+gmA-glxOs%3DBIFqlI-P4AXSVQ`1_q=SfPGZc8a ze}^X#T;q4!aHP#_{j68Xa8J~M(S39j?!Ms^qRB!}pW2hy7aS@WYs6<9Ds_VTrl%1x z>Z5P7Uw@%!;;r9F1IT5;k@PI~UB?J%oq4hh3m)YPs3)enE+?tFz@mH+Jjq8X9&`M= zJE!VRI8CY;P$T~UlHgloAo~8%Nj>CT^rbI;TW>PeNk6BjT~nLIrGN5jk|FE!A=KJvQZz#>yCAfrENj!K~r}{%=EjD!;iDtUou_Fr13S><{t?&*{;Pfpbuxsj1%e$+4c~&rXTt zvW_E)7GN2V{k{D~(}@z=J3=0f#Uf%gFGrb^gg5e3U22XYoANfzL@TpH1;=8xCiD%K zsVykh4mExvV_RC%@qP2Yu3ZF@>hGygVud#e3e5E91CkKpuh~JRO(mNck%5Z3L%GLvu}pG&Jx$h z!N}I-QXPBH(44aalIq>&mso>A1Lz;fpWE!m1WNj~#!~AY6>$_!8bCpv`gR75hDzVz z7*&IXMO!qFY9M0mgWMkW1d+-s#K|eftKrtuh8|+8ZXWKgzJ_eLYD1!Qz8~~Z2}wuU zBT=Y+KdBp$;br3S)A-pgHFY482rf(aHwaIfeZ!}BAQXbRHx}aSm(9u*)4?blgaq7f zx9Wtn%yxb5bJAH?K)J@~Z}2NsoQ`5glz4-F=M(CQV*k}bE7t_subqjFvUK86e+z`Z zvX^7Vpw%@S-jkVWg{F6JJ4a4s9xvTMKu`JapTt&A>VwF>P?YsCs*TP6=!T22&G1rl zP-1?H&$vif1sEl`4`zB<5!|Dn<6-u#vY$j`W+{h21Q;ib^gk--X0KU2{DC|N+ig3P z4Z$8oYTUJ@Z$=d6`}ot+Ge1u?J3@%2+c&qulT%@hklO8|>$rBeR2=~jIdsqBfEr;XDpZCcqx{F39Os%)HoST{}u={2~fuNxskZ33jDPWIaiPd<~ zNxF2v^yQ*x%x5@mMH$$cV3@phqZ7z87n%40v#aTasj3U`Co;HGCV0fejx%-CFA5>rl^lvZ}K-4sjK39c;|6$lqs8na(8c>Sk|HHev0h ztR&L|00961Z~??{JFgBQL5BGisd#5!Apy#B_?wkI??`HvP}+g#e4yv?8U(1ui)WMu z4|?CMF(r6&fAh7klB1S{dj~>{TAZz>7hq$w3UE_VGItoQQc6FVO~cqAy8G=~YsE%k zn*TUi`XCS(l_T+Fp~Eu+ed{rdu8Ea;QF>02vs5y}r{&D2g>4Rpbz!N)-ru|IVtEXO zdVI&2u~?Y{9Y41CkgtkU&O!9Z(^v%h)Nr>aUM~zG?o>m3%aD#}3XocGuw3+hJ`_|1 zaT)CI$DjzeS;;+pSlwB4oc7YVui^wa= II1gs9|AP9@yZ`_I literal 0 HcmV?d00001 diff --git a/workbench-for-google-cloud-workstations/conf/rserver.conf b/workbench-for-google-cloud-workstations/conf/rserver.conf index d9b416e4..2bd60929 100644 --- a/workbench-for-google-cloud-workstations/conf/rserver.conf +++ b/workbench-for-google-cloud-workstations/conf/rserver.conf @@ -6,6 +6,8 @@ server-project-sharing=0 auth-pam-sessions-enabled=1 rsession-no-profile=1 +server-license-manager-path=/opt/rstudio-license/license-manager + # Launcher Config launcher-address=127.0.0.1 launcher-port=5559 diff --git a/workbench-for-google-cloud-workstations/license-manager-shim b/workbench-for-google-cloud-workstations/license-manager-shim new file mode 100755 index 00000000..4ad2633d --- /dev/null +++ b/workbench-for-google-cloud-workstations/license-manager-shim @@ -0,0 +1,7 @@ +#!/bin/sh + +# Copyright (C) 2023 by Posit, PBC. + +dir=$(dirname "$0") +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/rstudio-license" +exec "$dir/license-manager-orig" "$@" diff --git a/workbench-for-google-cloud-workstations/startup.sh b/workbench-for-google-cloud-workstations/startup.sh index 76d3b19f..97421355 100644 --- a/workbench-for-google-cloud-workstations/startup.sh +++ b/workbench-for-google-cloud-workstations/startup.sh @@ -1,9 +1,11 @@ #!/bin/bash -export LICENSE_MANAGER_PATH=${LICENSE_MANAGER_PATH:-/usr/lib/rstudio-server/bin} +export LICENSE_MANAGER_PATH=${LICENSE_MANAGER_PATH:-/opt/rstudio-license} set -ex +exec >/startup.log 2>&1 + # Deactivate license when the process exits deactivate() { echo "== Exiting ==" From 20c885710410d8e7b52814e66c1214d0a7195f58 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 8 Jun 2023 12:15:51 -0700 Subject: [PATCH 57/86] Update env var --- workbench-for-google-cloud-workstations/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 1c4b5ad7..613c3553 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -23,7 +23,7 @@ ENV RSW_LAUNCHER_TIMEOUT 10 ENV DIAGNOSTIC_DIR /var/log/rstudio ENV DIAGNOSTIC_ENABLE false ENV DIAGNOSTIC_ONLY false -ENV LICENSE_MANAGER_PATH /usr/lib/rstudio-server/bin +ENV LICENSE_MANAGER_PATH /opt/rstudio-license ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter From 8b623d35a31e28cdb72003fad7f4aff7efaaae70 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 8 Jun 2023 12:18:33 -0700 Subject: [PATCH 58/86] Comment out exec logging --- workbench-for-google-cloud-workstations/startup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/startup.sh b/workbench-for-google-cloud-workstations/startup.sh index 97421355..b7d7cce2 100644 --- a/workbench-for-google-cloud-workstations/startup.sh +++ b/workbench-for-google-cloud-workstations/startup.sh @@ -4,7 +4,8 @@ export LICENSE_MANAGER_PATH=${LICENSE_MANAGER_PATH:-/opt/rstudio-license} set -ex -exec >/startup.log 2>&1 +# Uncomment to get a log for this script +#exec >/startup.log 2>&1 # Deactivate license when the process exits deactivate() { From 8dfcf64a8888a9a4efa18fb07e05827bdf464c7a Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 22 Jun 2023 10:08:43 -0700 Subject: [PATCH 59/86] Maximize build space for GCW GHA --- .github/workflows/build-workbench-for-gcw.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index d14cc164..c5af9184 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -21,6 +21,13 @@ jobs: IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: 'true' + - name: Check Out Repo uses: actions/checkout@v3 with: From ff7e23bdf4df2bca457f225f5d227fd1e3a34ea7 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 22 Jun 2023 10:20:01 -0700 Subject: [PATCH 60/86] Change buildx caching to a mounted directory with actual space --- workbench-for-google-cloud-workstations/Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 57a30908..028d3c8b 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -30,7 +30,7 @@ build *TAGS="": set -euxo pipefail BUILDX_ARGS="" if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max" + BUILDX_ARGS="--cache-from=type=local,src=~/.buildx-cache --cache-to=type=local,dest=~/.buildx-cache-new,mode=max" fi if [[ "{{TAGS}}" == "" ]]; then From 9c35403659e80b6c382900110127e20bec34ec94 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 22 Jun 2023 10:38:47 -0700 Subject: [PATCH 61/86] Dynamically set buildx caching path from var --- .github/workflows/build-workbench-for-gcw.yaml | 10 +++++----- workbench-for-google-cloud-workstations/Justfile | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index c5af9184..11b8776c 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -55,7 +55,7 @@ jobs: # - name: Cache Docker layers # uses: actions/cache@v3 # with: -# path: /tmp/.buildx-cache +# path: /${GITHUB_WORKFLOW}/.buildx-cache # key: ${{ runner.os }}-buildx-${{ github.sha }} # restore-keys: | # ${{ runner.os }}-buildx- @@ -104,7 +104,7 @@ jobs: docker tag ${{ steps.build.outputs.TAGS }} just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} -# - name: Move cache -# run: | -# rm -rf /tmp/.buildx-cache -# mv /tmp/.buildx-cache-new /tmp/.buildx-cache + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 028d3c8b..e3182c15 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -28,9 +28,10 @@ _make-default-tag: build *TAGS="": #!/usr/bin/env bash set -euxo pipefail + CACHE_PATH=${GITHUB_WORKSPACE:-/tmp} BUILDX_ARGS="" if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=~/.buildx-cache --cache-to=type=local,dest=~/.buildx-cache-new,mode=max" + BUILDX_ARGS="--cache-from=type=local,src=${CACHE_PATH}/.buildx-cache --cache-to=type=local,dest=${CACHE_PATH}/.buildx-cache-new,mode=max" fi if [[ "{{TAGS}}" == "" ]]; then From 2fac16ac8eab989c2b7bee98157994e3b27528ff Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 22 Jun 2023 11:24:12 -0700 Subject: [PATCH 62/86] Upgrade Workbench for GCW --- workbench-for-google-cloud-workstations/Dockerfile | 2 +- workbench-for-google-cloud-workstations/Justfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 613c3553..9da48727 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -9,7 +9,7 @@ ARG PYTHON_VERSION_ALT=3.9.16 ARG PYTHON_VERSION_JUPYTER=3.10.11 ARG QUARTO_VERSION=1.3.324 ARG DRIVERS_VERSION=2022.11.0 -ARG RSW_VERSION=2023.03.1+446.pro1 +ARG RSW_VERSION=2023.06.0+421.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index e3182c15..3a7a69c7 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -6,7 +6,7 @@ IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu2004" -RSW_VERSION := "2023.03.1+446.pro1" +RSW_VERSION := "2023.06.0+421.pro1" RSW_LICENSE := "" RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" From 09ae19a75d34637b298e43d8685ec0f0b6eceeb1 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 22 Jun 2023 12:57:05 -0700 Subject: [PATCH 63/86] Use build-push-action in GHA Revert version upgrade Update patch versions for Python Upgrade Pro Drivers --- .../workflows/build-workbench-for-gcw.yaml | 92 ++++++++++++------- .../Dockerfile | 10 +- .../Justfile | 10 +- 3 files changed, 70 insertions(+), 42 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 11b8776c..c7d06914 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -14,10 +14,10 @@ jobs: env: R_VERSION: 4.2.3 R_VERSION_ALT: 4.1.3 - PYTHON_VERSION: 3.10.11 - PYTHON_VERSION_ALT: 3.9.16 - RSW_VERSION: 2023.03.1-446.pro1 - #RSW_DOWNLOAD_URL: https://s3.amazonaws.com/rstudio-ide-build/server/bionic/amd64 + PYTHON_VERSION: 3.10.12 + PYTHON_VERSION_ALT: 3.9.17 + RSW_VERSION: 2023.03.1+446.pro1 + RSW_TAG_VERSION: 2023.03.1-446.pro1 IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench steps: @@ -68,26 +68,42 @@ jobs: # echo "VERSION=$VERSION" >> $GITHUB_OUTPUT # echo "TAG_SAFE_VERSION=${VERSION/+/-}" >> $GITHUB_OUTPUT - - name: Build base image - id: build - run: | - TAGS=$(just \ - BUILDX_PATH=${{ steps.buildx.outputs.name }} \ - RSW_VERSION=${{ env.RSW_VERSION }} \ - RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} \ - R_VERSION=${{ env.R_VERSION }} \ - R_VERSION_ALT=${{ env.R_VERSION_ALT }} \ - PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ - PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ - PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ - workbench-for-google-cloud-workstations/build \ - ${{ env.IMAGE_REGISTRY_NAME }}:latest ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_VERSION }}) - echo "TAGS=$TAGS" >> $GITHUB_OUTPUT + - name: Build + uses: docker/build-push-action@v4 + with: + context: ./workbench-for-google-cloud-workstations + load: true + cache-from: type=gha + cache-to: type=gha + build-args: | + RSW_VERSION=${{ env.RSW_VERSION }} + R_VERSION=${{ env.R_VERSION }} + R_VERSION_ALT=${{ env.R_VERSION_ALT }} + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} + PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} + tags: | + ${{ env.IMAGE_REGISTRY_NAME }}:latest, + ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} + +# - name: Build base image +# id: build +# run: | +# TAGS=$(just \ +# BUILDX_PATH=${{ steps.buildx.outputs.name }} \ +# RSW_VERSION=${{ env.RSW_VERSION }} \ +# RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} \ +# R_VERSION=${{ env.R_VERSION }} \ +# R_VERSION_ALT=${{ env.R_VERSION_ALT }} \ +# PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ +# PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ +# PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ +# workbench-for-google-cloud-workstations/build \ +# ${{ env.IMAGE_REGISTRY_NAME }}:latest ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_VERSION }}) +# echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - name: Test Image - continue-on-error: true run: | - IFS=" " read -a tags <<<"${{ steps.build.outputs.TAGS }}" just \ RSW_VERSION=${{ env.RSW_VERSION }} \ R_VERSION=${{ env.R_VERSION }} \ @@ -95,16 +111,28 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ - workbench-for-google-cloud-workstations/test "${tags[0]}" + workbench-for-google-cloud-workstations/test ${{ env.IMAGE_REGISTRY_NAME }}:latest - # FIXME: The second tag isn't persisting for some reason, having to reapply it here - - name: Push Image - run: | - docker image ls - docker tag ${{ steps.build.outputs.TAGS }} - just -f ci.Justfile push-images ${{ steps.build.outputs.TAGS }} + - name: Push + uses: docker/build-push-action@v4 + with: + push: true + context: workbench-for-google-cloud-workstations + load: true + cache-from: type=gha + cache-to: type=gha + build-args: | + RSW_VERSION=${{ env.RSW_VERSION }} + R_VERSION=${{ env.R_VERSION }} + R_VERSION_ALT=${{ env.R_VERSION_ALT }} + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} + PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} + tags: | + ${{ env.IMAGE_REGISTRY_NAME }}:latest, + ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache +# - name: Move cache +# run: | +# rm -rf /tmp/.buildx-cache +# mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 9da48727..6e774f34 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -4,12 +4,12 @@ FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.10.11 -ARG PYTHON_VERSION_ALT=3.9.16 -ARG PYTHON_VERSION_JUPYTER=3.10.11 +ARG PYTHON_VERSION=3.10.12 +ARG PYTHON_VERSION_ALT=3.9.17 +ARG PYTHON_VERSION_JUPYTER=3.10.12 ARG QUARTO_VERSION=1.3.324 -ARG DRIVERS_VERSION=2022.11.0 -ARG RSW_VERSION=2023.06.0+421.pro1 +ARG DRIVERS_VERSION=2023.05.0 +ARG RSW_VERSION=2023.03.1+446.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 3a7a69c7..9bb1d2d8 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -6,18 +6,18 @@ IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu2004" -RSW_VERSION := "2023.06.0+421.pro1" +RSW_VERSION := "2023.03.1+446.pro1" RSW_LICENSE := "" RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" -DRIVERS_VERSION := "2022.11.0" +DRIVERS_VERSION := "2023.05.0" R_VERSION := "4.2.3" R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.10.11" -PYTHON_VERSION_ALT := "3.9.16" -PYTHON_VERSION_JUPYTER := "3.10.11" +PYTHON_VERSION := "3.10.12" +PYTHON_VERSION_ALT := "3.9.17" +PYTHON_VERSION_JUPYTER := "3.10.12" QUARTO_VERSION := "1.3.324" From 55579d99a343c7cc3eab9389d46c83157b4091f5 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 22 Jun 2023 13:07:37 -0700 Subject: [PATCH 64/86] Remove maximize build space, may not be necessary --- .github/workflows/build-workbench-for-gcw.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index c7d06914..3c3175aa 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -21,12 +21,6 @@ jobs: IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - with: - root-reserve-mb: 512 - swap-size-mb: 1024 - remove-dotnet: 'true' - name: Check Out Repo uses: actions/checkout@v3 From 505336ac200d23f3e34afdea6e1d978262ad83a4 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 23 Jun 2023 07:54:03 -0700 Subject: [PATCH 65/86] Rollback python version upgrade, not yet released --- .github/workflows/build-workbench-for-gcw.yaml | 4 ++-- workbench-for-google-cloud-workstations/Dockerfile | 4 ++-- workbench-for-google-cloud-workstations/Justfile | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 3c3175aa..a0c7b395 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -14,8 +14,8 @@ jobs: env: R_VERSION: 4.2.3 R_VERSION_ALT: 4.1.3 - PYTHON_VERSION: 3.10.12 - PYTHON_VERSION_ALT: 3.9.17 + PYTHON_VERSION: 3.10.11 + PYTHON_VERSION_ALT: 3.9.16 RSW_VERSION: 2023.03.1+446.pro1 RSW_TAG_VERSION: 2023.03.1-446.pro1 IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 6e774f34..024c89f5 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -5,8 +5,8 @@ ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 ARG PYTHON_VERSION=3.10.12 -ARG PYTHON_VERSION_ALT=3.9.17 -ARG PYTHON_VERSION_JUPYTER=3.10.12 +ARG PYTHON_VERSION_ALT=3.9.16 +ARG PYTHON_VERSION_JUPYTER=3.10.11 ARG QUARTO_VERSION=1.3.324 ARG DRIVERS_VERSION=2023.05.0 ARG RSW_VERSION=2023.03.1+446.pro1 diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 9bb1d2d8..2b4b5d1c 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -15,9 +15,9 @@ DRIVERS_VERSION := "2023.05.0" R_VERSION := "4.2.3" R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.10.12" -PYTHON_VERSION_ALT := "3.9.17" -PYTHON_VERSION_JUPYTER := "3.10.12" +PYTHON_VERSION := "3.10.11" +PYTHON_VERSION_ALT := "3.9.16" +PYTHON_VERSION_JUPYTER := "3.10.11" QUARTO_VERSION := "1.3.324" From 1a7b2f39dc948cb3a039ebcdbba66565aa37513b Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 23 Jun 2023 08:17:46 -0700 Subject: [PATCH 66/86] Fix missed version --- workbench-for-google-cloud-workstations/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 024c89f5..380569fd 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -4,7 +4,7 @@ FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.10.12 +ARG PYTHON_VERSION=3.10.11 ARG PYTHON_VERSION_ALT=3.9.16 ARG PYTHON_VERSION_JUPYTER=3.10.11 ARG QUARTO_VERSION=1.3.324 From c9469d06f63f50adda67cea743693c3722482bfc Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 23 Jun 2023 09:32:40 -0700 Subject: [PATCH 67/86] Upgrade Workbench to 2023.06.0 and tick donwload url to focal --- workbench-for-google-cloud-workstations/Dockerfile | 4 ++-- workbench-for-google-cloud-workstations/Justfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 380569fd..8938b493 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -9,9 +9,9 @@ ARG PYTHON_VERSION_ALT=3.9.16 ARG PYTHON_VERSION_JUPYTER=3.10.11 ARG QUARTO_VERSION=1.3.324 ARG DRIVERS_VERSION=2023.05.0 -ARG RSW_VERSION=2023.03.1+446.pro1 +ARG RSW_VERSION=2023.06.0+421.pro1 ARG RSW_NAME=rstudio-workbench -ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 +ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 ENV RSW_LICENSE "" ENV RSW_LICENSE_SERVER "" diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 2b4b5d1c..8586a1c8 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -6,9 +6,9 @@ IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu2004" -RSW_VERSION := "2023.03.1+446.pro1" +RSW_VERSION := "2023.06.0+421.pro1" RSW_LICENSE := "" -RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" +RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/focal/amd64" RSW_LICENSE_SERVER := "" DRIVERS_VERSION := "2023.05.0" From 1fcaf39e3acce5288a34cd78dc98e96c1f952a80 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 23 Jun 2023 09:55:21 -0700 Subject: [PATCH 68/86] Explicitly define download url in build workflow Add .env file --- .../workflows/build-workbench-for-gcw.yaml | 47 +++---------------- workbench-for-google-cloud-workstations/.env | 10 ++++ 2 files changed, 17 insertions(+), 40 deletions(-) create mode 100644 workbench-for-google-cloud-workstations/.env diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index a0c7b395..92621c03 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -18,14 +18,14 @@ jobs: PYTHON_VERSION_ALT: 3.9.16 RSW_VERSION: 2023.03.1+446.pro1 RSW_TAG_VERSION: 2023.03.1-446.pro1 + RSW_DOWNLOAD_URL: https://download2.rstudio.org/server/focal/amd64 + DRIVERS_VERSION: 2023.05.0 IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench steps: - name: Check Out Repo uses: actions/checkout@v3 - with: - ref: gcw-workbench - name: Set up Just uses: extractions/setup-just@v1 @@ -45,23 +45,6 @@ jobs: run: | gcloud auth configure-docker -q us-central1-docker.pkg.dev -# FIXME(ianpittwood): Caching keeps overrunning our disk space -# - name: Cache Docker layers -# uses: actions/cache@v3 -# with: -# path: /${GITHUB_WORKFLOW}/.buildx-cache -# key: ${{ runner.os }}-buildx-${{ github.sha }} -# restore-keys: | -# ${{ runner.os }}-buildx- - -# - name: Get Version -# id: get-version -# run: | -# VERSION=`just -f ci.Justfile get-version workbench --type=release --local` -# VERSION=${VERSION//\"/} -# echo "VERSION=$VERSION" >> $GITHUB_OUTPUT -# echo "TAG_SAFE_VERSION=${VERSION/+/-}" >> $GITHUB_OUTPUT - - name: Build uses: docker/build-push-action@v4 with: @@ -76,26 +59,12 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} + DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} + RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} tags: | ${{ env.IMAGE_REGISTRY_NAME }}:latest, ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} -# - name: Build base image -# id: build -# run: | -# TAGS=$(just \ -# BUILDX_PATH=${{ steps.buildx.outputs.name }} \ -# RSW_VERSION=${{ env.RSW_VERSION }} \ -# RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} \ -# R_VERSION=${{ env.R_VERSION }} \ -# R_VERSION_ALT=${{ env.R_VERSION_ALT }} \ -# PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ -# PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ -# PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ -# workbench-for-google-cloud-workstations/build \ -# ${{ env.IMAGE_REGISTRY_NAME }}:latest ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_VERSION }}) -# echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - - name: Test Image run: | just \ @@ -105,6 +74,7 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ + DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} \ workbench-for-google-cloud-workstations/test ${{ env.IMAGE_REGISTRY_NAME }}:latest - name: Push @@ -122,11 +92,8 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} + DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} + RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} tags: | ${{ env.IMAGE_REGISTRY_NAME }}:latest, ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} - -# - name: Move cache -# run: | -# rm -rf /tmp/.buildx-cache -# mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/workbench-for-google-cloud-workstations/.env b/workbench-for-google-cloud-workstations/.env new file mode 100644 index 00000000..971e7573 --- /dev/null +++ b/workbench-for-google-cloud-workstations/.env @@ -0,0 +1,10 @@ +RSW_VERSION=2023.06.0+421.pro1 +RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 +RSW_NAME=rstudio-workbench +PYTHON_VERSION=3.10.11 +PYTHON_VERSION_ALT=3.9.16 +PYTHON_VERSION_JUPYTER=3.9.16 +R_VERSION_ALT=4.1.3 +R_VERSION=4.2.3 +DRIVERS_VERSION=2023.05.0 +QUARTO_VERSION=1.3.324 From 4a995041711b43e56d51180d92c18493b527c595 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 23 Jun 2023 10:45:34 -0700 Subject: [PATCH 69/86] Upgrade workflow to 2023.06.0 --- .github/workflows/build-workbench-for-gcw.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 92621c03..82809f4c 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -16,8 +16,8 @@ jobs: R_VERSION_ALT: 4.1.3 PYTHON_VERSION: 3.10.11 PYTHON_VERSION_ALT: 3.9.16 - RSW_VERSION: 2023.03.1+446.pro1 - RSW_TAG_VERSION: 2023.03.1-446.pro1 + RSW_VERSION: 2023.06.0+421.pro1 + RSW_TAG_VERSION: 2023.06.0-421.pro1 RSW_DOWNLOAD_URL: https://download2.rstudio.org/server/focal/amd64 DRIVERS_VERSION: 2023.05.0 IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench From 07821a21efbffa2b4459ef777636b2db7327e326 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 23 Jun 2023 11:33:09 -0700 Subject: [PATCH 70/86] Increase timeout for R lib goss tests Use RSW version for test tag --- .github/workflows/build-workbench-for-gcw.yaml | 6 +++--- workbench-for-google-cloud-workstations/test/goss.yaml | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 82809f4c..b9208f40 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -62,7 +62,7 @@ jobs: DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} tags: | - ${{ env.IMAGE_REGISTRY_NAME }}:latest, + ${{ env.IMAGE_REGISTRY_NAME }}:latest ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} - name: Test Image @@ -75,7 +75,7 @@ jobs: PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} \ - workbench-for-google-cloud-workstations/test ${{ env.IMAGE_REGISTRY_NAME }}:latest + workbench-for-google-cloud-workstations/test ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} - name: Push uses: docker/build-push-action@v4 @@ -95,5 +95,5 @@ jobs: DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} tags: | - ${{ env.IMAGE_REGISTRY_NAME }}:latest, + ${{ env.IMAGE_REGISTRY_NAME }}:latest ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index 92c270cc..674665ef 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -224,8 +224,10 @@ command: {{- range $r_package_list }} Check R {{ $r_version }} has "{{.}}" installed: exec: /opt/R/{{$r_version}}/bin/R --slave -e "library(\"{{.}}\")" + timeout: 60000 exit-status: 0 Check R {{ $r_version_alt }} has "{{.}}" installed: exec: /opt/R/{{$r_version_alt}}/bin/R --slave -e "library(\"{{.}}\")" + timeout: 60000 exit-status: 0 {{end}} \ No newline at end of file From 978a8c1a24a09d9d70141a8effdc90c778e10d80 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 23 Jun 2023 11:44:29 -0700 Subject: [PATCH 71/86] Remove "load" from push action --- .github/workflows/build-workbench-for-gcw.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index b9208f40..43c64f04 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -82,7 +82,6 @@ jobs: with: push: true context: workbench-for-google-cloud-workstations - load: true cache-from: type=gha cache-to: type=gha build-args: | From d707e49544435f38f66dc5698645418660a7f34f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 28 Jun 2023 15:38:37 -0700 Subject: [PATCH 72/86] Rollback to 2023.03.2 --- .github/workflows/build-workbench-for-gcw.yaml | 6 +++--- workbench-for-google-cloud-workstations/Dockerfile | 4 ++-- workbench-for-google-cloud-workstations/Justfile | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index 43c64f04..d1ecb988 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -16,9 +16,9 @@ jobs: R_VERSION_ALT: 4.1.3 PYTHON_VERSION: 3.10.11 PYTHON_VERSION_ALT: 3.9.16 - RSW_VERSION: 2023.06.0+421.pro1 - RSW_TAG_VERSION: 2023.06.0-421.pro1 - RSW_DOWNLOAD_URL: https://download2.rstudio.org/server/focal/amd64 + RSW_VERSION: 2023.03.2+454.pro2 + RSW_TAG_VERSION: 2023.03.2-454.pro2 + RSW_DOWNLOAD_URL: https://download2.rstudio.org/server/bionic/amd64 DRIVERS_VERSION: 2023.05.0 IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 8938b493..08acb7c0 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -9,9 +9,9 @@ ARG PYTHON_VERSION_ALT=3.9.16 ARG PYTHON_VERSION_JUPYTER=3.10.11 ARG QUARTO_VERSION=1.3.324 ARG DRIVERS_VERSION=2023.05.0 -ARG RSW_VERSION=2023.06.0+421.pro1 +ARG RSW_VERSION=2023.03.2+454.pro2 ARG RSW_NAME=rstudio-workbench -ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 +ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 ENV RSW_LICENSE "" ENV RSW_LICENSE_SERVER "" diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 8586a1c8..5b351780 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -6,9 +6,9 @@ IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu2004" -RSW_VERSION := "2023.06.0+421.pro1" +RSW_VERSION := "2023.03.2+454.pro2" RSW_LICENSE := "" -RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/focal/amd64" +RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" DRIVERS_VERSION := "2023.05.0" From 224883650b83ec425d562a85cbdb34fd5b7d15d3 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 29 Jun 2023 07:09:26 -0700 Subject: [PATCH 73/86] Remove unnecessary launcher keys --- .../.gitignore | 2 ++ .../conf/launcher.pem | 28 ------------------- .../conf/launcher.pub | 9 ------ 3 files changed, 2 insertions(+), 37 deletions(-) create mode 100644 workbench-for-google-cloud-workstations/.gitignore delete mode 100644 workbench-for-google-cloud-workstations/conf/launcher.pem delete mode 100644 workbench-for-google-cloud-workstations/conf/launcher.pub diff --git a/workbench-for-google-cloud-workstations/.gitignore b/workbench-for-google-cloud-workstations/.gitignore new file mode 100644 index 00000000..1bdfee60 --- /dev/null +++ b/workbench-for-google-cloud-workstations/.gitignore @@ -0,0 +1,2 @@ +conf/launcher.pem +conf/launcher.pub diff --git a/workbench-for-google-cloud-workstations/conf/launcher.pem b/workbench-for-google-cloud-workstations/conf/launcher.pem deleted file mode 100644 index f63cf133..00000000 --- a/workbench-for-google-cloud-workstations/conf/launcher.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDAT7k2z06gPhLx -RNLavdwFTHG+KTbJouQeHWjvFY+NFyePINp4r6R+J3z+Wq/LJ4usUvafx4qTTNhL -Iub6Uix+fRbRwuRv8K0y+nOj7nszulRF7Q/3CRWCLoVzNKHiLbbdHma5MpNtH9gz -Cry87PGk8ZQviIt2pFqsI0fiu7wSPdUmMpt2ssBTduOkuNJ8c0aH+UK7uBGQt6yy -q/BLNKiUMc4Krb/1xKxXkzcyhs5OTtfg95fMOItvZtijliLFW/vDnzbGk6IqEonf -ZOwKUB3cLChIGtdkyQ6KzjTRbtzc0D/XVnOuhFsLrYPqoQcap1IqFhhZbzviR1c4 -fOGpnG5lAgMBAAECggEANQfG8y5HaDi10IJCAr/IJE3oL5/906xvaz6A6Szw4N85 -sWZF2HHW0lLhTOKFB2hoQ6S9DWJg/C18l3uQZgay7k5gNIehppyfpicaLGJe5uzC -wGWWuXBv1e4uLo6c9VnVyzPO79LahIT7gVe8o/xSxkE3s8PbcWPw/Pbdo9BRFFYu -Gcm7BxJXvgFW9V1SP1fVT3o93n9vFZa3Ti6uH+XyQqtCasZp4jAmO5aHzDl0zIu8 -iV8l0KdrmoSNuxBOe8tkIU+j1F/orQhX7GJRmHG/vIfOSnonf2wqNGHyeikehzYy -6zjxujN2saYVedP3T1Uuox/4TJUoB8pWRCVjIIbWIQKBgQDqg8QYgC9dJ1Cbl+SX -j8I1lDasPbPjoUvkHxmR6XqQWTIJKai5uuITYMfFRmPxj6vLc7Ot6cR1ndpglX+4 -NM98m99Mquzx6raNunUvZbOZC7U1B2KbXOcH5UWvQWyum9yy78ysfZRxFYCeZErO -0Rvf7vDCfabvVW3jZHRse/4OEwKBgQDR7iKEFbb3gl/Hq/1CVoHbeQk6Ak3f5ZfM -IimYiY86OBZ5qMyF0yoZa8qakR5tmgaIN9ZXIddMEtJF+JVuXA2TCKU13X8REebZ -vXCAirneBmngv6MDzKz3kRN5Q+q2dw2yvBSHzFhF6J9A3ZODJeWuZuiohVoVEH+6 -HGVH4MHApwKBgDROCTmRtX6oWyEo4pIvPRioIv2NkYaunfeWPCTAF1cEndVPVlN+ -bBf4WPYV+gqRibaJ6sRaRDgbae5qyZ6zIKfhOA+oe8qFGAxKTwKwQztaUiMur36J -XPn01/L7u5rCJpo+Q78hDy+dpR8urUStfPMYHtiyH753ple2OwVLLmHDAoGANbIe -fAB940MispToW5566qND2+WY+6rjL+CRGSYeT3rnLMkjPxurCgOLOAvkZ45dG1yE -45aRVjD+Pc1+k1LSWgh9EZENHlWg4ZLWAtr4/bRQQp9h1+fzWgNuorbfgzpic5Qu -ACohbgg5K8NBJut7mydRXUF2sCqfMQBwW3k+tcMCgYAFeSEptO8zfmYI3Q1U84R8 -MvxcSef3GBpL3ZuSIvWeNN3WYKluSn+VxYVUWg2i3/4AuBWqbY6bxdOVJElEcqHS -ALXnZCwMel6A/59Fp86eY4S3Mqj1MMfONwu1h96x3WSxazfzIRdFLTCI5N9J03sR -CXhynMytlSaH5GkThW5N4w== ------END PRIVATE KEY----- diff --git a/workbench-for-google-cloud-workstations/conf/launcher.pub b/workbench-for-google-cloud-workstations/conf/launcher.pub deleted file mode 100644 index 2c5babe0..00000000 --- a/workbench-for-google-cloud-workstations/conf/launcher.pub +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwE+5Ns9OoD4S8UTS2r3c -BUxxvik2yaLkHh1o7xWPjRcnjyDaeK+kfid8/lqvyyeLrFL2n8eKk0zYSyLm+lIs -fn0W0cLkb/CtMvpzo+57M7pURe0P9wkVgi6FczSh4i223R5muTKTbR/YMwq8vOzx -pPGUL4iLdqRarCNH4ru8Ej3VJjKbdrLAU3bjpLjSfHNGh/lCu7gRkLessqvwSzSo -lDHOCq2/9cSsV5M3MobOTk7X4PeXzDiLb2bYo5YixVv7w582xpOiKhKJ32TsClAd -3CwoSBrXZMkOis400W7c3NA/11ZzroRbC62D6qEHGqdSKhYYWW874kdXOHzhqZxu -ZQIDAQAB ------END PUBLIC KEY----- From d8642b778a1a543a1fbc1854113f9d310aa49618 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 29 Jun 2023 08:52:15 -0700 Subject: [PATCH 74/86] Update workbench-for-google-cloud-workstations/Justfile Co-authored-by: Benjamin R. J. Schwedler --- workbench-for-google-cloud-workstations/Justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 5b351780..849437ff 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -1,3 +1,4 @@ +set dotenv-load := true set positional-arguments BUILDX_PATH := "" From 1582be17ecb4fcc3acd6b294e1a1b4b9690caaeb Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 6 Jul 2023 07:58:45 -0700 Subject: [PATCH 75/86] Use .env file for Justfile and GHA workflow --- .../workflows/build-workbench-for-gcw.yaml | 13 ++--- workbench-for-google-cloud-workstations/.env | 10 ++-- .../Justfile | 55 +++++++------------ 3 files changed, 31 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index d1ecb988..a89dac06 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -12,21 +12,16 @@ jobs: build-workbench-for-gcw: runs-on: ubuntu-latest env: - R_VERSION: 4.2.3 - R_VERSION_ALT: 4.1.3 - PYTHON_VERSION: 3.10.11 - PYTHON_VERSION_ALT: 3.9.16 - RSW_VERSION: 2023.03.2+454.pro2 - RSW_TAG_VERSION: 2023.03.2-454.pro2 - RSW_DOWNLOAD_URL: https://download2.rstudio.org/server/bionic/amd64 - DRIVERS_VERSION: 2023.05.0 - IMAGE_REGISTRY_NAME: us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench + DOTENV_FILE: ./workbench-for-google-cloud-workstations/.env steps: - name: Check Out Repo uses: actions/checkout@v3 + - name: Set environment variables using .env file + run: cat ${{ env.DOTENV_FILE }} >> "$GITHUB_ENV" + - name: Set up Just uses: extractions/setup-just@v1 env: diff --git a/workbench-for-google-cloud-workstations/.env b/workbench-for-google-cloud-workstations/.env index 971e7573..c1788575 100644 --- a/workbench-for-google-cloud-workstations/.env +++ b/workbench-for-google-cloud-workstations/.env @@ -1,10 +1,12 @@ -RSW_VERSION=2023.06.0+421.pro1 -RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 +RSW_VERSION=2023.03.2+454.pro2 +RSW_TAG_VERSION=2023.03.2-454.pro2 +RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 RSW_NAME=rstudio-workbench PYTHON_VERSION=3.10.11 PYTHON_VERSION_ALT=3.9.16 -PYTHON_VERSION_JUPYTER=3.9.16 -R_VERSION_ALT=4.1.3 +PYTHON_VERSION_JUPYTER=3.10.11 R_VERSION=4.2.3 +R_VERSION_ALT=4.1.3 DRIVERS_VERSION=2023.05.0 QUARTO_VERSION=1.3.324 +IMAGE_REGISTRY_NAME=us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 849437ff..1c239567 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -1,29 +1,16 @@ -set dotenv-load := true +set dotenv-load set positional-arguments BUILDX_PATH := "" -IMAGE_PREFIX := "rstudio-" PRODUCT := "workbench" IMAGE_OS := "ubuntu2004" -RSW_VERSION := "2023.03.2+454.pro2" RSW_LICENSE := "" -RSW_DOWNLOAD_URL := "https://download2.rstudio.org/server/bionic/amd64" RSW_LICENSE_SERVER := "" -DRIVERS_VERSION := "2023.05.0" - -R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.3" - -PYTHON_VERSION := "3.10.11" -PYTHON_VERSION_ALT := "3.9.16" -PYTHON_VERSION_JUPYTER := "3.10.11" - -QUARTO_VERSION := "1.3.324" _make-default-tag: - echo "{{IMAGE_PREFIX}}{{PRODUCT}}:$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" + echo "${IMAGE_REGISTRY_NAME}:$(just -f ../Justfile _get-tag-safe-version "${RSW_VERSION}")" # Build Workbench image - just build 2022.07.2+576.pro12 rstudio/rstudio-workbench-gcw:2022.07.2 build *TAGS="": @@ -36,7 +23,7 @@ build *TAGS="": fi if [[ "{{TAGS}}" == "" ]]; then - raw_tag_array=($(just IMAGE_PREFIX="{{IMAGE_PREFIX}}" RSW_VERSION="{{RSW_VERSION}}" _make-default-tag)) + raw_tag_array=($(just _make-default-tag)) else raw_tag_array=("{{TAGS}}") fi @@ -49,15 +36,15 @@ build *TAGS="": docker buildx --builder="{{ BUILDX_PATH }}" build --load ${BUILDX_ARGS} \ ${tag_array[@]} \ - --build-arg RSW_VERSION="{{ RSW_VERSION }}" \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION_JUPYTER="{{ PYTHON_VERSION_JUPYTER }}" \ - --build-arg QUARTO_VERSION="{{ QUARTO_VERSION }}" \ - --build-arg DRIVERS_VERSION="{{ DRIVERS_VERSION }}" \ - --build-arg RSW_DOWNLOAD_URL="{{ RSW_DOWNLOAD_URL }}" \ + --build-arg RSW_VERSION="${RSW_VERSION}" \ + --build-arg R_VERSION="${R_VERSION}" \ + --build-arg R_VERSION_ALT="${R_VERSION_ALT}" \ + --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ + --build-arg PYTHON_VERSION_ALT="${PYTHON_VERSION_ALT}" \ + --build-arg PYTHON_VERSION_JUPYTER="${PYTHON_VERSION_JUPYTER}" \ + --build-arg QUARTO_VERSION="${QUARTO_VERSION}" \ + --build-arg DRIVERS_VERSION="${DRIVERS_VERSION}" \ + --build-arg RSW_DOWNLOAD_URL="${RSW_DOWNLOAD_URL}" \ --file=./Dockerfile . echo ${raw_tag_array[@]} @@ -66,18 +53,18 @@ build *TAGS="": test TAG=`just _make-default-tag` CMD="": #!/usr/bin/env bash set -euxo pipefail - RSW_VERSION_CLEAN=$(sed "s/daily-/daily+/" <<<{{RSW_VERSION}}) + RSW_VERSION_CLEAN=$(sed "s/daily-/daily+/" <<<"${RSW_VERSION}") IMAGE_NAME="{{ TAG }}" \ - RSW_VERSION="$RSW_VERSION_CLEAN" \ + RSW_VERSION="${RSW_VERSION_CLEAN}" \ RSW_LICENSE="{{ RSW_LICENSE }}" \ RSW_LICENSE_SERVER="{{ RSW_LICENSE_SERVER }}" \ - DRIVERS_VERSION="{{ DRIVERS_VERSION }}" \ - R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - PYTHON_VERSION_JUPYTER="{{ PYTHON_VERSION_JUPYTER }}" \ - QUARTO_VERSION="{{ QUARTO_VERSION }}" \ + DRIVERS_VERSION="${DRIVERS_VERSION}" \ + R_VERSION="${R_VERSION}" \ + R_VERSION_ALT="${R_VERSION_ALT}" \ + PYTHON_VERSION="${PYTHON_VERSION}" \ + PYTHON_VERSION_ALT="${PYTHON_VERSION_ALT}" \ + PYTHON_VERSION_JUPYTER="${PYTHON_VERSION_JUPYTER}" \ + QUARTO_VERSION="${QUARTO_VERSION}" \ docker-compose -f ./docker-compose.test.yml run sut {{ CMD }} # Test Workbench image interactively - just test-i rstudio/rstudio-workbench:ubuntu1804-2022.07.2-576.pro12 2022.07.2+576.pro12 From 824245ade60c75731022bd95382db5e23403f95f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 6 Jul 2023 09:25:29 -0700 Subject: [PATCH 76/86] Fix test step in workflow --- .github/workflows/build-workbench-for-gcw.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml index a89dac06..59825fd1 100644 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ b/.github/workflows/build-workbench-for-gcw.yaml @@ -62,15 +62,7 @@ jobs: - name: Test Image run: | - just \ - RSW_VERSION=${{ env.RSW_VERSION }} \ - R_VERSION=${{ env.R_VERSION }} \ - R_VERSION_ALT=${{ env.R_VERSION_ALT }} \ - PYTHON_VERSION=${{ env.PYTHON_VERSION }} \ - PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} \ - PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} \ - DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} \ - workbench-for-google-cloud-workstations/test ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} + just workbench-for-google-cloud-workstations/test ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} - name: Push uses: docker/build-push-action@v4 From ef74e2832bf6f3e745145037d62e3d6d02c4d479 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 6 Jul 2023 09:26:01 -0700 Subject: [PATCH 77/86] Revise README.md from original copy to GCW specific copy --- .../README.md | 240 ++---------------- 1 file changed, 25 insertions(+), 215 deletions(-) diff --git a/workbench-for-google-cloud-workstations/README.md b/workbench-for-google-cloud-workstations/README.md index 6d3f94a4..55eb644a 100644 --- a/workbench-for-google-cloud-workstations/README.md +++ b/workbench-for-google-cloud-workstations/README.md @@ -1,14 +1,10 @@ -# Quick reference +_# Quick reference * Maintained by: [the Posit Docker team](https://github.com/rstudio/rstudio-docker-products) * Where to get help: [our Github Issues page](https://github.com/rstudio/rstudio-docker-products/issues) * RStudio Workbench image: [Docker Hub](https://hub.docker.com/r/rstudio/rstudio-workbench) * RStudio r-session-complete image: [Docker Hub](https://hub.docker.com/r/rstudio/r-session-complete) - -# Supported tags and respective Dockerfile links - -* [`2022.07.2`, `bionic`, `ubuntu1804`, `bionic-2022.07.2`, `ubuntu1804-2022.07.2`](https://github.com/rstudio/rstudio-docker-products/blob/main/workbench/Dockerfile.ubuntu1804) -* [`jammy`, `ubuntu2204`, `jammy-2022.07.2`, `ubuntu2204-2022.07.2`](https://github.com/rstudio/rstudio-docker-products/blob/main/workbench/Dockerfile.ubuntu2204) +* Registry for this image: [Posit's Google Cloud Artifact Registry](https://console.cloud.google.com/artifacts/docker/posit-images/us-central1/cloud-workstations/workbench) # What is RStudio Workbench? @@ -31,242 +27,56 @@ Some of the functionality that Workbench provides is: * Auditing and monitoring * Advanced R and Python session management -For more information on running RStudio Workbench in your organization please visit https://www.rstudio.com/products/workbench/. +For more information on running RStudio Workbench in your organization please visit +https://www.rstudio.com/products/workbench/. # Notice for support -1. This image may introduce **BREAKING** changes; as such we recommend: - - Avoid using the `{operating-system}` tags to avoid unexpected version changes, and - - Always read through the [NEWS](./NEWS.md) to understand the changes before updating. -1. Outdated images will be removed periodically from DockerHub as product version updates are made. Please make plans to - update at times or use your own build of the images. -1. These images are meant as a starting point for your needs. Consider creating a fork of this repo, where you can - continue to merge in changes we make while having your own security scanning, base OS in use, or other custom - changes. We - provide [instructions for building](https://github.com/rstudio/rstudio-docker-products#instructions-for-building) for - these cases. +1. This image is still in early development. Some bugs may be present, and we may introduce **BREAKING** changes in + order to improve your user experience; as such we recommend: + - Always read through the [NEWS](./NEWS.md) to understand the changes before updating or when encountering a bug. + - Use the `latest` or "version" tags. Avoid using `daily` tagged images unless advised to by Posit staff. +1. Outdated images will be removed periodically from GCAR as product version updates are made. Please make plans to + update at times, use the `latest` or version tag, or use your own build of the images. # How to use this image -To verify basic functionality as a first step: - -``` -# Replace with valid license -export RSW_LICENSE=XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX +This image is designed for exclusive use with [Google Cloud Workstations](https://cloud.google.com/workstations). For +the generalized version of the Workbench image, go [here](../workbench). -# Run without persistent data using default configuration -docker run -it \ - -p 8787:8787 \ - -e RSW_LICENSE=$RSW_LICENSE \ - rstudio/rstudio-workbench:ubuntu1804 -``` +Using Google Cloud Workstations requires a Google Cloud Platform account. Click +[here](https://console.cloud.google.com/workstations/overview) to navigate to the Cloud Workstations console. Posit +Workbench is provided in the default list of "Code editors on base images" when creating a workstation configuration. +Alternatively, administrators can use the "Custom container image" option to reference a specific tag from +[Posit's public Google Cloud Artifact Registry](https://console.cloud.google.com/artifacts/docker/posit-images/us-central1/cloud-workstations/workbench). - -Open [http://localhost:8787](http://localhost:8787) to access RStudio Workbench. The default username and password are -`rstudio`. +For more information, please see +[Cloud Workstation's official documentation](https://cloud.google.com/workstations/docs/develop-code-using-posit-workbench-rstudio). ## Overview -Note that running the RStudio Workbench Docker image requires a valid RStudio Workbench license. +Note that running the RStudio Workbench Docker image requires a valid RStudio Workbench license for Google Cloud +Workstations. Licenses can be obtained by emailing sales@posit.co. This container includes: 1. Two versions of R 2. Two versions of Python -3. RStudio Connect - -## Configuration - -RStudio Workbench is configured via config files in the `/etc/rstudio` directory. Mount this directory as -a volume from the host machine. Changes will take effect when the container is restarted. - -You can review possible RStudio Workbench configuration [in the documentation](https://docs.rstudio.com/ide/workbench/). - -See a complete example of server configuration at `workbench/conf`. - -### Persistent Data - -In order to persist user files between container restarts please mount the `/home` directory from a persistent volume on the host -machine or your docker orchestration system. +3. RStudio Workbench ### Licensing -The RStudio Workbench Docker image requires a valid license, which can be set in three ways: - -1. Setting the `RSW_LICENSE` environment variable to a valid license key inside the container -2. Setting the `RSW_LICENSE_SERVER` environment variable to a valid license server / port inside the container -3. Mounting a `/etc/rstudio-server/license.lic` single file that contains a valid license for RStudio Workbench - -**NOTE:** the "offline activation process" is not supported by this image today. Offline installations will need -to explore using a license server, license file, or custom image with manual intervention. - -### User Provisioning - -By default, the container will create a test user, which you can control or disable with the environment -variables: `RSW_TESTUSER`, `RSW_TESTUSER_PASSWD`, `RSW_TESTUSER_UID`. - -#### sssd / LDAP / Active Directory - -If you have a directory (LDAP server, Active Directory, etc.) available to -provision users, `sssd` is installed in the container and enabled by default ( -see `Process Management` below). In order to make use of it, you will need to -mount your own configuration file into `/etc/sssd/conf.d/`. For instance, - -_sssd.conf_ -```ini -[sssd] -config_file_version = 2 -domains = LDAP - -[domain/LDAP] -id_provider = ldap -auth_provider = ldap -chpass_provider = ldap -sudo_provider = ldap -# ... more configuration -``` - -Then: -```bash -# sssd is picky about file permissions -chmod 600 sssd.conf - -docker run -it \ - -p 8787:8787 -p 5559:5559 \ - -v $PWD/data/rsp:/home \ - -v $PWD/server-pro/conf/:/etc/rstudio \ - -v $PWD/sssd.conf:/etc/sssd/conf.d/sssd.conf \ - -e RSP_LICENSE=$RSP_LICENSE \ - rstudio/rstudio-workbench:ubuntu1804 -``` - -It is worth noting that you may also need to modify the PAM configuration files -in the container, if you are using PAM for custom authentication or session -behavior. See the [RStudio Workbench -guide](https://docs.rstudio.com/ide/server-pro/authenticating-users.html) for -more information. +The RStudio Workbench Docker image requires a valid license, which can be set using the `RSW_LICENSE` environment +variable to a valid license key inside the container. ### Environment variables | Variable | Description | Default | |-----|---|-----------| -| `RSW_TESTUSER` | Test user to be created in the container, turn off with an empty value | `rstudio` | -| `RSW_TESTUSER_PASSWD` | Test user password | `rstudio` | -| `RSW_TESTUSER_UID` | Test user UID | `10000` | | `RSW_LICENSE` | License key for RStudio Workbench, format should be: `XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX` | None | -| `RSW_LICENSE_SERVER` | Floating license server, format should be: `my.url.com:port` | None | | `RSW_LAUNCHER` | Whether or not to use launcher locally / start the launcher process | true | | `RSW_LAUNCHER_TIMEOUT` | The timeout, in seconds, to wait for launcher to start listening on the expected port before failing startup | 10 | -### Ports - -| Variable | Description | -|--------|---| -| `8787` | Default HTTP Port for RStudio Connect | -| `5559` | Port for RStudio Launcher server | - -### Example usage: - -```bash -# Replace with valid license -export RSW_LICENSE=XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX - -# Run without persistent data and using an external configuration -docker run -it \ - -p 8787:8787 -p 5559:5559 \ - -v $PWD/workbench/conf/:/etc/rstudio \ - -e RSW_LICENSE=$RSW_LICENSE \ - rstudio/rstudio-workbench:ubuntu1804 - -# Run with persistent data and using an external configuration -docker run -it \ - -p 8787:8787 -p 5559:5559 \ - -v $PWD/data/rsw:/home \ - -v $PWD/workbench/conf/:/etc/rstudio \ - -e RSW_LICENSE=$RSW_LICENSE \ - rstudio/rstudio-workbench:ubuntu1804 -``` - -Open [http://localhost:8787](http://localhost:8787) to access RStudio Workbench. -The default username and password are `rstudio`. - -### Process Management - -In order for RStudio Workbench to function properly, several services need to -be accounted for. We run these services using -[`supervisord`](http://supervisord.org/). `supervisord` is an open source -process supervisor with an active community. It enables running multiple -services in the container, as well as exiting the container if _any_ of those -services exit. - -> NOTE: generally speaking, running multiple services in a single container is an anti-pattern. -> However, we have implemented the below as a workaround until RStudio Workbench can -> handle users and other processes in a more container-friendly fashion - -Details on the various processes and their configuration is below: - -- **RStudio Workbench**: the main server process - - this startup configuration is mounted at `/startup/base` - -- **RStudio Job Launcher**: enables launching Jupyter, JupyterLab, and VSCode - sessions, as well as talking to job schedulers like Slurm and Kubernetes. - - Optional and enabled by default - - this startup configuration is mounted at `/startup/launcher` - - to disable, mount an empty volume over `/startup/launcher` - -- **sssd**: often used for user provisioning when connected to an LDAP - directory or other user store. - - Optional, and enabled by default, but with a "dummy" domain so it does - nothing. - - To use this with your directory, mount required `.conf` files into - `/etc/sssd/conf.d/` (more details in `User Provisioning`, above) - - Startup configuration is installed at `/startup/user-provisioning/` - - To disable entirely, mount an empty volume over - `/startup/user-provisioning/` - -- **custom**: Do you have a service that you need to run inside the container - for user provisioning or otherwise? Mount other configuration files into - `/startup/custom`, and they will be started and managed by `supervisord` as - well - - NOTE: in many cases (i.e. Kubernetes) `initContainers` or `sidecar` containers are a better fit - -## Caveats of product licensing in containers - -*Note: This section **does not** apply to activations using license files.* - -There is currently a known licensing bug when using our products in containers. If the container is not stopped -gracefully, the license deactivation step may fail or be skipped. Failing to deactivate the license can result in a -"license leak" where a product activation is used up and cannot be deactivated using traditional methods as the -activation state on the container has been lost. - -To avoid "leaking" licenses, we encourage users not to force kill containers and to use `--stop-timeout 120` and -`--time 120` for `docker run` and `docker stop` commands respectively. This helps ensure the deactivation script has -ample time to run properly. - -In some situations, it can be difficult or impossible to avoid a hard termination (e.g. power failure, -critical error on host). Unfortunately, any of these cases can still cause a license to leak an activation. To help -prevent a license leak in these situations, users can mount the following directories to persistent storage to preserve -the license state data across restarts of the container. **These directories differ between products.** - -* License Key - * `/var/lib/.local` - * `/var/lib/.prof` - * `/var/lib/rstudio-workbench` -* Floating License - * `/var/lib/.TurboFloat` - -Please note that the files created in these directories are hardware locked and non-transferable between hosts. Due to -the nature of the hardware fingerprinting algorithm, any low-level changes to the host or container can cause existing -license state files to invalidate. To avoid this problem, we advise that product containers are gracefully shutdown -and allowed to deactivate prior to changing any hardware or firmware on the host (e.g. upgrading a network card or -updating BIOS) or the container (e.g. changing the network driver used or the allocated number of CPU cores). - -While preserving license state data *can* help avoid license leaks across restarts, it's not a guarantee. If you run -into issues with your license, please do not hesitate to [contact Posit support](https://support.posit.co/hc/en-us). - -While neither of these solutions will eliminate the problem, they should help mitigate it. We are still investigating a -long-term solution. - # Licensing The license associated with the RStudio Docker Products repository is located [in LICENSE.md](https://github.com/rstudio/rstudio-docker-products/blob/main/LICENSE.md). @@ -276,4 +86,4 @@ licenses (i.e. bash, linux, system libraries, etc., along with any other direct software being contained). It is an image user's responsibility to ensure that use of this image (and any of its dependent layers) complies with -all relevant licenses for the software contained in the image. +all relevant licenses for the software contained in the image._ From 94d05713e81f200d4ff02f22624069dc014ad551 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 6 Jul 2023 11:47:01 -0700 Subject: [PATCH 78/86] Clean up caching - Purge pip caches after install steps - Purge apt caches where missing - Consolidate COPY steps --- .../Dockerfile | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 08acb7c0..86b7be31 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -24,11 +24,8 @@ ENV DIAGNOSTIC_DIR /var/log/rstudio ENV DIAGNOSTIC_ENABLE false ENV DIAGNOSTIC_ONLY false ENV LICENSE_MANAGER_PATH /opt/rstudio-license - ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter -COPY --chmod=755 TurboActivate.dat /opt/rstudio-license/license-manager.conf - ### Update/upgrade system packages ### RUN apt-get update --fix-missing \ && apt-get upgrade -yq \ @@ -76,7 +73,8 @@ RUN curl -O https://cdn.rstudio.com/r/ubuntu-2004/pkgs/r-${R_VERSION}_1_amd64.de && rm -f ./r-${R_VERSION}_1_amd64.deb \ && rm -f ./r-${R_VERSION_ALT}_1_amd64.deb \ && ln -s /opt/R/${R_VERSION}/bin/R /usr/local/bin/R \ - && ln -s /opt/R/${R_VERSION}/bin/Rscript /usr/local/bin/Rscript + && ln -s /opt/R/${R_VERSION}/bin/Rscript /usr/local/bin/Rscript \ + && rm -rf /var/lib/apt/lists/* ### Install Python versions ### RUN curl -O https://cdn.rstudio.com/python/ubuntu-2004/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ @@ -89,14 +87,19 @@ RUN curl -O https://cdn.rstudio.com/python/ubuntu-2004/pkgs/python-${PYTHON_VERS && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade pip \ + && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade pip + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade pip \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ + && rm -rf /var/lib/apt/lists/* ### Install basic data science packages for Python and R ### COPY deps/* / RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /py_packages.txt \ + && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install -r /py_packages.txt \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ && ./install_r_packages.sh \ && rm install_r_packages.sh py_packages.txt r_packages.txt @@ -108,10 +111,11 @@ ENV LC_ALL en_US.UTF-8 ### Install Quarto ### RUN curl -o quarto-linux-amd64.deb -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb \ + && apt-get update \ && apt-get install -yq ./quarto-linux-amd64.deb \ + && rm -rf /var/lib/apt/lists/* \ && rm quarto-linux-amd64.deb - ### Install Pro Drivers ### RUN apt-get update \ && apt-get install -yq --no-install-recommends unixodbc unixodbc-dev \ @@ -146,13 +150,6 @@ RUN mkdir -p /opt/rstudio-license/ \ && mv /opt/rstudio-license/license-manager /opt/rstudio-license/license-manager-orig \ && rm -f /usr/lib/rstudio-server/bin/license-manager -COPY --chmod=755 license-manager-shim /opt/rstudio-license/license-manager -COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh -COPY startup-launcher/* /startup/launcher/ -COPY startup-user-provisioning/* /startup/user-provisioning/ -COPY startup/* /startup/base/ -COPY supervisord.conf /etc/supervisor/supervisord.conf - ### Install Jupyter and extensions ### RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyter \ && /opt/python/jupyter/bin/pip install \ @@ -169,20 +166,30 @@ RUN /opt/python/"${PYTHON_VERSION_JUPYTER}"/bin/python -m venv /opt/python/jupyt && /opt/python/jupyter/bin/jupyter-nbextension enable --sys-prefix --py rsconnect_jupyter \ && /opt/python/jupyter/bin/jupyter-serverextension enable --sys-prefix --py rsconnect_jupyter \ && /opt/python/jupyter/bin/python -m ipykernel install --name py${PYTHON_VERSION} --display-name "Python ${PYTHON_VERSION}" \ - && /opt/python/jupyter/bin/python -m ipykernel install --name py${PYTHON_VERSION_ALT} --display-name "Python ${PYTHON_VERSION_ALT}" + && /opt/python/jupyter/bin/python -m ipykernel install --name py${PYTHON_VERSION_ALT} --display-name "Python ${PYTHON_VERSION_ALT}" \ + && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/rstudio/wait-for-it/master/wait-for-it.sh \ && chmod +x /usr/local/bin/wait-for-it.sh -COPY --chmod=600 sssd.conf /etc/sssd/sssd.conf -COPY conf/* /etc/rstudio/ -COPY --chmod=600 conf/launcher.p* /etc/rstudio - RUN mkdir -p /var/lib/rstudio-server/monitor/log \ && chown -R rstudio-server:rstudio-server /var/lib/rstudio-server/monitor \ && mkdir -p /startup/custom/ \ && printf '\n# allow home directory creation\nsession required pam_mkhomedir.so skel=/etc/skel umask=0022' >> /etc/pam.d/common-session +COPY --chmod=755 TurboActivate.dat /opt/rstudio-license/license-manager.conf +COPY --chmod=755 license-manager-shim /opt/rstudio-license/license-manager +COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh +COPY startup-launcher/* /startup/launcher/ +COPY startup-user-provisioning/* /startup/user-provisioning/ +COPY startup/* /startup/base/ +COPY supervisord.conf /etc/supervisor/supervisord.conf +COPY --chmod=600 sssd.conf /etc/sssd/sssd.conf +COPY conf/* /etc/rstudio/ +COPY --chmod=600 conf/launcher.p* /etc/rstudio + +# GCW specific COPY --chmod=755 workstation-startup/* /etc/workstation-startup.d/ COPY --chmod=644 jupyter/jupyter_notebook_config.json /opt/python/jupyter/etc/jupyter/jupyter_notebook_config.json From 88525c1b41080fc50c230c8b84e5752506344fde Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 6 Jul 2023 12:40:46 -0700 Subject: [PATCH 79/86] Remove `-x` flag from `just run` target --- workbench-for-google-cloud-workstations/Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 1c239567..4514a15d 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -74,7 +74,7 @@ test-i TAG=`just _make-default-tag`: # Run Workbench - just RSW_LICENSE="" run rstudio/r-session-complete:ubuntu1804-2022.07.2-576.pro12 run TAG=`just _make-default-tag` CMD="": #!/usr/bin/env bash - set -euxo pipefail + set -euo pipefail if [ -z "{{ RSW_LICENSE }}" ] && [ -z "{{ RSW_LICENSE_SERVER }}" ]; then echo "Please set RSW_LICENSE or RSW_LICENSE_SERVER before running." exit 1 From 7d1b98d7d3a5e2b98311a9840c8e60ee0c95c238 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 27 Jul 2023 13:17:06 -0700 Subject: [PATCH 80/86] Update versions for bug fixes --- workbench-for-google-cloud-workstations/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile index 86b7be31..a927c7ed 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile @@ -4,10 +4,10 @@ FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.10.11 -ARG PYTHON_VERSION_ALT=3.9.16 -ARG PYTHON_VERSION_JUPYTER=3.10.11 -ARG QUARTO_VERSION=1.3.324 +ARG PYTHON_VERSION=3.10.12 +ARG PYTHON_VERSION_ALT=3.9.17 +ARG PYTHON_VERSION_JUPYTER=3.10.12 +ARG QUARTO_VERSION=1.3.340 ARG DRIVERS_VERSION=2023.05.0 ARG RSW_VERSION=2023.03.2+454.pro2 ARG RSW_NAME=rstudio-workbench From 040059475ee6d9d0616374026a7e9ac3fc31713d Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 8 Aug 2023 11:05:52 -0700 Subject: [PATCH 81/86] Rebase and merge WGCW workflow into release workflow --- .../actions/build-test-scan-push/action.yaml | 16 ++++ .github/workflows/build-release.yaml | 83 ++++++++++++++++++ .../workflows/build-workbench-for-gcw.yaml | 85 ------------------- workbench-for-google-cloud-workstations/.env | 8 +- .../{Dockerfile => Dockerfile.ubuntu2004} | 2 +- .../Justfile | 23 ++++- 6 files changed, 126 insertions(+), 91 deletions(-) delete mode 100644 .github/workflows/build-workbench-for-gcw.yaml rename workbench-for-google-cloud-workstations/{Dockerfile => Dockerfile.ubuntu2004} (99%) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 333344b4..338facd6 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -45,6 +45,10 @@ inputs: description: Username for authentication with DockerHub required: true type: string + gcp-json: + description: JSON for authenticating Google Cloud Platform + default: "" + type: string runs: using: "composite" @@ -71,6 +75,18 @@ runs: username: ${{ inputs.dockerhub-username }} password: ${{ inputs.dockerhub-token }} + - name: Authenticate to Google Cloud + continue-on-error: true + uses: google-github-actions/auth@v1 + with: + credentials_json: '${{ inputs.gcp-json }}' + + - name: Authenticate GCAR + run: | + if [[ "${{ inputs.gcp-json }}" != "" ]]; then + gcloud auth configure-docker -q us-central1-docker.pkg.dev + fi + - name: Build id: image-build uses: docker/build-push-action@v4 diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 205c4f72..13295850 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -318,3 +318,86 @@ jobs: dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} # End retry logic + + build-workbench-for-google-cloud-workstations: + needs: [ build-base, build-base-pro ] + runs-on: ubuntu-latest + name: build-workbench-for-google-cloud-workstations + + concurrency: + group: build-products-${{ matrix.config.product }}-${{ matrix.config.os }}-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Get Version + id: get-version + run: | + VERSION=$(just workbench-for-google-cloud-workstations/get-version) + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + + - name: Get build args + id: get-build-args + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + BUILD_ARGS=$(just workbench-for-google-cloud-workstations/get-build-args) + echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT + echo "$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + + - name: Get tags + id: get-tags + run: | + IMAGE_TAGS=$(just workbench-for-google-cloud-workstations/get-build-tags) + echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + + - name: Build/Test/Scan/Push base pro image + id: build1 + uses: ./.github/actions/build-test-scan-push + continue-on-error: true + with: + context: ./workbench-for-google-cloud-workstations + os: ubuntu2004 + product: workbench-for-google-cloud-workstations + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + # Begin retry logic + + - name: Wait 60s on failure before retrying + if: steps.build1.outcome == 'failure' + run: sleep 60 + + - name: Retry - Build/Test/Scan/Push base pro image + id: build2 + if: steps.build1.outcome == 'failure' + uses: ./.github/actions/build-test-scan-push + with: + context: ./workbench-for-google-cloud-workstations + os: ubuntu2004 + product: workbench-for-google-cloud-workstations + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + + # End retry logic diff --git a/.github/workflows/build-workbench-for-gcw.yaml b/.github/workflows/build-workbench-for-gcw.yaml deleted file mode 100644 index 59825fd1..00000000 --- a/.github/workflows/build-workbench-for-gcw.yaml +++ /dev/null @@ -1,85 +0,0 @@ -on: - schedule: - - cron: "0 8 * * 1" - push: - branches: - - gcw-workbench - - main - - dev - -name: Build Workbench for Google Cloud Workstations -jobs: - build-workbench-for-gcw: - runs-on: ubuntu-latest - env: - DOTENV_FILE: ./workbench-for-google-cloud-workstations/.env - - steps: - - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set environment variables using .env file - run: cat ${{ env.DOTENV_FILE }} >> "$GITHUB_ENV" - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v1 - with: - credentials_json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' - - - name: Authenticate GCAR - run: | - gcloud auth configure-docker -q us-central1-docker.pkg.dev - - - name: Build - uses: docker/build-push-action@v4 - with: - context: ./workbench-for-google-cloud-workstations - load: true - cache-from: type=gha - cache-to: type=gha - build-args: | - RSW_VERSION=${{ env.RSW_VERSION }} - R_VERSION=${{ env.R_VERSION }} - R_VERSION_ALT=${{ env.R_VERSION_ALT }} - PYTHON_VERSION=${{ env.PYTHON_VERSION }} - PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} - PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} - DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} - RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} - tags: | - ${{ env.IMAGE_REGISTRY_NAME }}:latest - ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} - - - name: Test Image - run: | - just workbench-for-google-cloud-workstations/test ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} - - - name: Push - uses: docker/build-push-action@v4 - with: - push: true - context: workbench-for-google-cloud-workstations - cache-from: type=gha - cache-to: type=gha - build-args: | - RSW_VERSION=${{ env.RSW_VERSION }} - R_VERSION=${{ env.R_VERSION }} - R_VERSION_ALT=${{ env.R_VERSION_ALT }} - PYTHON_VERSION=${{ env.PYTHON_VERSION }} - PYTHON_VERSION_ALT=${{ env.PYTHON_VERSION_ALT }} - PYTHON_VERSION_JUPYTER=${{ env.PYTHON_VERSION }} - DRIVERS_VERSION=${{ env.DRIVERS_VERSION }} - RSW_DOWNLOAD_URL=${{ env.RSW_DOWNLOAD_URL }} - tags: | - ${{ env.IMAGE_REGISTRY_NAME }}:latest - ${{ env.IMAGE_REGISTRY_NAME }}:${{ env.RSW_TAG_VERSION }} diff --git a/workbench-for-google-cloud-workstations/.env b/workbench-for-google-cloud-workstations/.env index c1788575..1a28dfba 100644 --- a/workbench-for-google-cloud-workstations/.env +++ b/workbench-for-google-cloud-workstations/.env @@ -2,11 +2,11 @@ RSW_VERSION=2023.03.2+454.pro2 RSW_TAG_VERSION=2023.03.2-454.pro2 RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 RSW_NAME=rstudio-workbench -PYTHON_VERSION=3.10.11 -PYTHON_VERSION_ALT=3.9.16 -PYTHON_VERSION_JUPYTER=3.10.11 +PYTHON_VERSION=3.10.12 +PYTHON_VERSION_ALT=3.9.17 +PYTHON_VERSION_JUPYTER=3.10.12 R_VERSION=4.2.3 R_VERSION_ALT=4.1.3 DRIVERS_VERSION=2023.05.0 -QUARTO_VERSION=1.3.324 +QUARTO_VERSION=1.3.340 IMAGE_REGISTRY_NAME=us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench diff --git a/workbench-for-google-cloud-workstations/Dockerfile b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 similarity index 99% rename from workbench-for-google-cloud-workstations/Dockerfile rename to workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 index a927c7ed..4edcf9cb 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 @@ -1,4 +1,4 @@ -FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest +FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:public-image-current ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive diff --git a/workbench-for-google-cloud-workstations/Justfile b/workbench-for-google-cloud-workstations/Justfile index 4514a15d..060f4f47 100644 --- a/workbench-for-google-cloud-workstations/Justfile +++ b/workbench-for-google-cloud-workstations/Justfile @@ -12,6 +12,27 @@ RSW_LICENSE_SERVER := "" _make-default-tag: echo "${IMAGE_REGISTRY_NAME}:$(just -f ../Justfile _get-tag-safe-version "${RSW_VERSION}")" +get-version: + echo "${RSW_VERSION}" + +get-build-args: + #!/usr/bin/env bash + printf "RSW_VERSION=${RSW_VERSION} + R_VERSION=${R_VERSION} + R_VERSION_ALT=${R_VERSION_ALT} + PYTHON_VERSION=${PYTHON_VERSION} + PYTHON_VERSION_ALT=${PYTHON_VERSION_ALT} + PYTHON_VERSION_JUPYTER=${PYTHON_VERSION_JUPYTER} + QUARTO_VERSION=${QUARTO_VERSION} + DRIVERS_VERSION=${DRIVERS_VERSION} + SRC_IMAGE_NAME=${SRC_IMAGE_NAME} + RSW_DOWNLOAD_URL=${RSW_DOWNLOAD_URL}" + +get-build-tags: + #!/usr/bin/env bash + echo ${IMAGE_REGISTRY_NAME}:${RSW_TAG_VERSION},\ + ${IMAGE_REGISTRY_NAME}:latest + # Build Workbench image - just build 2022.07.2+576.pro12 rstudio/rstudio-workbench-gcw:2022.07.2 build *TAGS="": #!/usr/bin/env bash @@ -45,7 +66,7 @@ build *TAGS="": --build-arg QUARTO_VERSION="${QUARTO_VERSION}" \ --build-arg DRIVERS_VERSION="${DRIVERS_VERSION}" \ --build-arg RSW_DOWNLOAD_URL="${RSW_DOWNLOAD_URL}" \ - --file=./Dockerfile . + --file=./Dockerfile.ubuntu2004 . echo ${raw_tag_array[@]} From 29939e5e02c185fe975099fbc4c64f5041a9dcf9 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 8 Aug 2023 11:08:05 -0700 Subject: [PATCH 82/86] Fix missing shell --- .github/actions/build-test-scan-push/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 338facd6..dd9f461a 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -82,6 +82,7 @@ runs: credentials_json: '${{ inputs.gcp-json }}' - name: Authenticate GCAR + shell: bash run: | if [[ "${{ inputs.gcp-json }}" != "" ]]; then gcloud auth configure-docker -q us-central1-docker.pkg.dev From 2602e5cbe1604fadc5c1e9b70b5499d12a395e48 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 8 Aug 2023 11:45:25 -0700 Subject: [PATCH 83/86] Fix bash logic --- .github/actions/build-test-scan-push/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index dd9f461a..55d64603 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -84,7 +84,7 @@ runs: - name: Authenticate GCAR shell: bash run: | - if [[ "${{ inputs.gcp-json }}" != "" ]]; then + if [ ! -z "${{ inputs.gcp-json }}" ]; then gcloud auth configure-docker -q us-central1-docker.pkg.dev fi From 0351692e3ca3028afbea5476c2cef21fa143a46e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 9 Aug 2023 08:14:32 -0700 Subject: [PATCH 84/86] Do not trace licensing in startup.sh --- workbench-for-google-cloud-workstations/startup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/workbench-for-google-cloud-workstations/startup.sh b/workbench-for-google-cloud-workstations/startup.sh index b7d7cce2..21fb32ce 100644 --- a/workbench-for-google-cloud-workstations/startup.sh +++ b/workbench-for-google-cloud-workstations/startup.sh @@ -25,6 +25,8 @@ verify_installation(){ rstudio-server verify-installation --verify-user=$RSW_TESTUSER | tee $DIAGNOSTIC_DIR/verify.log } +set +x + # Support RSP_ or RSW_ prefix RSP_LICENSE=${RSP_LICENSE:-${RSW_LICENSE}} RSP_LICENSE_SERVER=${RSP_LICENSE_SERVER:-${RSW_LICENSE_SERVER}} @@ -45,6 +47,8 @@ unset RSP_LICENSE_SERVER unset RSW_LICENSE unset RSW_LICENSE_SERVER +set -x + # Start Launcher if [ "$RSW_LAUNCHER" == "true" ]; then echo "Waiting for launcher to startup... to disable set RSW_LAUNCHER=false" From e1c1679b578cc0b37eb7db7668b55f289df3b7e4 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 9 Aug 2023 08:15:42 -0700 Subject: [PATCH 85/86] Update patch in launcher-env --- workbench-for-google-cloud-workstations/conf/launcher-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/conf/launcher-env b/workbench-for-google-cloud-workstations/conf/launcher-env index 6cbec885..4a53e727 100644 --- a/workbench-for-google-cloud-workstations/conf/launcher-env +++ b/workbench-for-google-cloud-workstations/conf/launcher-env @@ -3,4 +3,4 @@ Environment: LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 JobType: any -Environment: PATH=/opt/python/3.9.16/bin:$PATH +Environment: PATH=/opt/python/3.9.17/bin:$PATH From 42069d738c7a9fab933a45c864916509cda6854f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 9 Aug 2023 08:19:33 -0700 Subject: [PATCH 86/86] Move apt package list to deps/apt_packages.txt --- .../Dockerfile.ubuntu2004 | 39 +++---------------- .../deps/apt_packages.txt | 32 +++++++++++++++ 2 files changed, 37 insertions(+), 34 deletions(-) create mode 100644 workbench-for-google-cloud-workstations/deps/apt_packages.txt diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 index 4edcf9cb..23c8c64c 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 @@ -26,42 +26,14 @@ ENV DIAGNOSTIC_ONLY false ENV LICENSE_MANAGER_PATH /opt/rstudio-license ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter +### Copy package lists and install scripts ### +COPY deps/* / + ### Update/upgrade system packages ### RUN apt-get update --fix-missing \ && apt-get upgrade -yq \ - && apt-get install -yq --no-install-recommends \ - apt-transport-https \ - build-essential \ - ca-certificates \ - default-jdk \ - dirmngr \ - dpkg-sig \ - gnupg \ - krb5-user \ - libcap2 \ - libcurl4-gnutls-dev \ - libglib2.0-0 \ - libicu-dev \ - libnss-sss \ - libpam-sss \ - libpq-dev \ - libpq5 \ - libsecret-1-dev \ - libsm6 \ - libssl-dev \ - libuser \ - libuser1-dev \ - libxext6 \ - libxrender1 \ - locales \ - oddjob-mkhomedir \ - rrdtool \ - sssd \ - supervisor \ - tcl \ - tk \ - tk-dev \ - tk-table \ + && xargs -a /apt_packages.txt apt-get install -yq --no-install-recommends \ + && rm /apt_packages.txt \ && rm -rf /var/lib/apt/lists/* ### Install R versions ### @@ -95,7 +67,6 @@ RUN curl -O https://cdn.rstudio.com/python/ubuntu-2004/pkgs/python-${PYTHON_VERS && rm -rf /var/lib/apt/lists/* ### Install basic data science packages for Python and R ### -COPY deps/* / RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /py_packages.txt \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install -r /py_packages.txt \ diff --git a/workbench-for-google-cloud-workstations/deps/apt_packages.txt b/workbench-for-google-cloud-workstations/deps/apt_packages.txt new file mode 100644 index 00000000..bf6d9c10 --- /dev/null +++ b/workbench-for-google-cloud-workstations/deps/apt_packages.txt @@ -0,0 +1,32 @@ +apt-transport-https +build-essential +ca-certificates +default-jdk +dirmngr +dpkg-sig +gnupg +krb5-user +libcap2 +libcurl4-gnutls-dev +libglib2.0-0 +libicu-dev +libnss-sss +libpam-sss +libpq-dev +libpq5 +libsecret-1-dev +libsm6 +libssl-dev +libuser +libuser1-dev +libxext6 +libxrender1 +locales +oddjob-mkhomedir +rrdtool +sssd +supervisor +tcl +tk +tk-dev +tk-table