diff --git a/.gitignore b/.gitignore index 8873881..3a5e5e7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ venv/ .coverage tests/fixtures/projects/metadata/ tests/fixtures/projects/output/ +.Rhistory diff --git a/opensafely/__init__.py b/opensafely/__init__.py index 4825306..5ddc630 100644 --- a/opensafely/__init__.py +++ b/opensafely/__init__.py @@ -22,6 +22,7 @@ info, jupyter, pull, + rstudio, unzip, upgrade, ) @@ -141,6 +142,7 @@ def add_subcommand(cmd, module): add_subcommand("info", info) add_subcommand("exec", execute) add_subcommand("clean", clean) + add_subcommand("rstudio", rstudio) warn_if_updates_needed(sys.argv) args = parser.parse_args() diff --git a/opensafely/jupyter.py b/opensafely/jupyter.py index 3c806f6..3121c38 100644 --- a/opensafely/jupyter.py +++ b/opensafely/jupyter.py @@ -1,14 +1,9 @@ import argparse import json import os -import socket import subprocess -import sys -import threading import time -import webbrowser from pathlib import Path -from urllib import request from opensafely import utils @@ -16,21 +11,6 @@ DESCRIPTION = "Run a jupyter lab notebook using the OpenSAFELY environment" -# poor mans debugging because debugging threads on windows is hard -if os.environ.get("DEBUG", False): - - def debug(msg): - # threaded output for some reason needs the carriage return or else - # it doesn't reset the cursor. - sys.stderr.write("DEBUG: " + msg.replace("\n", "\r\n") + "\r\n") - sys.stderr.flush() - -else: - - def debug(msg): - pass - - def add_arguments(parser): parser.add_argument( "--directory", @@ -56,7 +36,7 @@ def add_arguments(parser): "--port", "-p", default=None, - help="Port to run on", + help="Port to run on (random by default)", ) parser.add_argument( "jupyter_args", @@ -66,67 +46,42 @@ def add_arguments(parser): ) -def open_browser(name, port): +def get_metadata(name): + """Read the login token from the generated json file in the container""" + metadata = None + metadata_path = "/tmp/.local/share/jupyter/runtime/nbserver-*.json" + + # wait for jupyter to be set up + start = time.time() + while metadata is None and time.time() - start < 120.0: + ps = subprocess.run( + ["docker", "exec", name, "bash", "-c", f"cat {metadata_path}"], + text=True, + capture_output=True, + ) + if ps.returncode == 0: + utils.debug(ps.stdout) + metadata = json.loads(ps.stdout) + else: + time.sleep(1) + + if metadata is None: + utils.debug("get_metadata: Could not get metadata") + return None + + return metadata + + +def read_metadata_and_open(name, port): try: - metadata = None - metadata_path = "/tmp/.local/share/jupyter/runtime/nbserver-*.json" - - # wait for jupyter to be set up - start = time.time() - while metadata is None and time.time() - start < 120.0: - ps = subprocess.run( - ["docker", "exec", name, "bash", "-c", f"cat {metadata_path}"], - text=True, - capture_output=True, - ) - if ps.returncode == 0: - debug(ps.stdout) - metadata = json.loads(ps.stdout) - else: - time.sleep(1) - - if metadata is None: - debug("open_browser: Could not get metadata") - return - - url = f"http://localhost:{port}/?token={metadata['token']}" - debug(f"open_browser: url={url}") - - # wait for port to be open - debug("open_browser: waiting for port") - start = time.time() - while time.time() - start < 60.0: - try: - response = request.urlopen(url, timeout=1) - except (request.URLError, OSError): - pass - else: - break - - if not response: - debug("open_browser: open_browser: could not get response") - return - - # open a webbrowser pointing to the docker container - debug("open_browser: open_browser: opening browser window") - webbrowser.open(url, new=2) - - except Exception: - # reformat exception printing to work from thread - import traceback - - sys.stderr.write("Error in open browser thread:\r\n") - tb = traceback.format_exc().replace("\n", "\r\n") - sys.stderr.write(tb) - sys.stderr.flush() - - -def get_free_port(): - sock = socket.socket() - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - sock.close() - return port + metadata = get_metadata(name) + if metadata: + url = f"http://localhost:{port}/?token={metadata['token']}" + utils.open_browser(url) + else: + utils.debug("could not retrieve login token from jupyter container") + except Exception as exc: + utils.print_exception_from_thread(exc) def main(directory, name, port, no_browser, jupyter_args): @@ -134,10 +89,7 @@ def main(directory, name, port, no_browser, jupyter_args): name = f"os-jupyter-{directory.name}" if port is None: - # this is a race condition, as something else could consume the socket - # before docker binds to it, but the chance of that on a user's - # personal machine is very small. - port = str(get_free_port()) + port = str(utils.get_free_port()) jupyter_cmd = [ "jupyter", @@ -154,13 +106,6 @@ def main(directory, name, port, no_browser, jupyter_args): print(f"Running following jupyter cmd in OpenSAFELY docker container {name}...") print(" ".join(jupyter_cmd)) - if not no_browser: - # start thread to open web browser - thread = threading.Thread(target=open_browser, args=(name, port), daemon=True) - thread.name = "browser thread" - debug("starting open_browser thread") - thread.start() - docker_args = [ # we use our port on both sides of the docker port mapping so that # jupyter's logging uses the correct port from the user's perspective @@ -174,9 +119,14 @@ def main(directory, name, port, no_browser, jupyter_args): "PYTHONPATH=/workspace", ] - debug("docker: " + " ".join(docker_args)) + if not no_browser: + utils.open_in_thread(read_metadata_and_open, (name, port)) + + utils.debug("docker: " + " ".join(docker_args)) + ps = utils.run_docker( docker_args, "python", jupyter_cmd, interactive=True, directory=directory ) + # we want to exit with the same code that jupyter did return ps.returncode diff --git a/opensafely/rstudio.py b/opensafely/rstudio.py new file mode 100644 index 0000000..decc7ca --- /dev/null +++ b/opensafely/rstudio.py @@ -0,0 +1,91 @@ +import os +import subprocess +from pathlib import Path +from sys import platform + +from opensafely import utils + + +DESCRIPTION = "Run an RStudio Server session using the OpenSAFELY environment" + + +def add_arguments(parser): + parser.add_argument( + "--directory", + "-d", + default=os.getcwd(), + type=Path, + help="Directory to run the RStudio Server session in (default is current dir)", + ) + parser.add_argument( + "--name", help="Name of docker image (defaults to use directory name)" + ) + parser.add_argument( + "--port", + "-p", + default=None, + help="Port to run on (random by default)", + ) + + +def main(directory, name, port): + if name is None: + name = f"os-rstudio-{directory.name}" + + if port is None: + port = str(utils.get_free_port()) + + url = f"http://localhost:{port}" + + if platform == "linux": + uid = os.getuid() + else: + uid = None + + # check for rstudio image, if not present pull image + imgchk = subprocess.run( + ["docker", "image", "inspect", "ghcr.io/opensafely-core/rstudio:latest"], + capture_output=True, + ) + if imgchk.returncode == 1: + subprocess.run( + [ + "docker", + "pull", + "--platform=linux/amd64", + "ghcr.io/opensafely-core/rstudio:latest", + ], + check=True, + ) + + docker_args = [ + f"-p={port}:8787", + f"--name={name}", + f"--hostname={name}", + f"--env=HOSTPLATFORM={platform}", + f"--env=HOSTUID={uid}", + ] + + gitconfig = Path.home() / ".gitconfig" + if gitconfig.exists(): + docker_args.append(f"--volume={gitconfig}:/home/rstudio/local-gitconfig") + + utils.debug("docker: " + " ".join(docker_args)) + print( + f"Opening an RStudio Server session at {url}. " + "When you are finished working please press Ctrl+C here to end the session" + ) + + utils.open_in_thread(utils.open_browser, (url,)) + + ps = utils.run_docker( + docker_args, + image="rstudio", + interactive=True, + # rstudio needs to start as root, but drops privileges to uid later + user="0:0", + directory=directory, + ) + + # we want to exit with the same code that rstudio-server did + return ps.returncode diff --git a/opensafely/utils.py b/opensafely/utils.py index b487b6c..aa28ff0 100644 --- a/opensafely/utils.py +++ b/opensafely/utils.py @@ -3,12 +3,32 @@ import os import pathlib import shutil +import socket import subprocess import sys +import threading +import time +import webbrowser +from urllib import request from opensafely._vendor.jobrunner import config +# poor mans debugging because debugging threads on windows is hard +if os.environ.get("DEBUG", False): + + def debug(msg): + # threaded output for some reason needs the carriage return or else + # it doesn't reset the cursor. + sys.stderr.write("DEBUG: " + msg.replace("\n", "\r\n") + "\r\n") + sys.stderr.flush() + +else: + + def debug(msg): + pass + + def get_default_user(): try: # On ~unix, in order for any files that get created to have the @@ -72,7 +92,7 @@ def git_bash_tty_wrapper(): def run_docker( docker_args, image, - cmd, + cmd=(), directory=None, interactive=False, user=None, @@ -97,6 +117,9 @@ def run_docker( "--rm", "--init", "--label=opensafely", + # all our docker images are this platform + # helps when running on M-series macs. + "--platform=linux/amd64", ] if interactive: @@ -128,3 +151,59 @@ def run_docker( print(" ".join(docker_cmd)) return subprocess.run(docker_cmd, *args, **kwargs) + + +def get_free_port(): + """Get a port that is free on the users host machine""" + # this is a race condition, as something else could consume the socket + # before docker binds to it, but the chance of that on a user's + # personal machine is very small. + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + return port + + +def print_exception_from_thread(exc): + # reformat exception printing to work from thread + import traceback + + sys.stderr.write("Error in background thread:\r\n") + tb = traceback.format_exc(exc).replace("\n", "\r\n") + sys.stderr.write(tb) + sys.stderr.flush() + + +def open_browser(url): + try: + debug(f"open_browser: url={url}") + + # wait for port to be open + debug("open_browser: waiting for port") + start = time.time() + while time.time() - start < 60.0: + try: + response = request.urlopen(url, timeout=1) + except (request.URLError, OSError): + pass + else: + break + + if not response: + debug("open_browser: open_browser: could not get response") + return + + # open a webbrowser pointing to the docker container + debug("open_browser: open_browser: opening browser window") + webbrowser.open(url, new=2) + + except Exception as exc: + print_exception_from_thread(exc) + + +def open_in_thread(target, args): + thread = threading.Thread(target=target, args=args, daemon=True) + thread.name = "browser thread" + debug("starting browser thread") + thread.start() diff --git a/tests/test_execute.py b/tests/test_execute.py index df997fc..bb75dec 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -15,6 +15,7 @@ def test_execute_main_args(run, no_user): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", f"--volume={pathlib.Path.cwd()}://workspace", "--env", @@ -41,6 +42,7 @@ def test_execute_main_entrypoint(run, no_user): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", f"--volume={pathlib.Path.cwd()}://workspace", "--env", @@ -65,6 +67,7 @@ def test_execute_main_env(run, no_user): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", f"--volume={pathlib.Path.cwd()}://workspace", "--env", @@ -94,6 +97,7 @@ def test_execute_main_env_in_env(run, no_user): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", f"--volume={pathlib.Path.cwd()}://workspace", "--env", @@ -120,6 +124,7 @@ def test_execute_main_user_cli_arg_overrides(default, run, monkeypatch): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", "--user=1234:5678", f"--volume={pathlib.Path.cwd()}://workspace", @@ -146,6 +151,7 @@ def test_execute_main_user_linux_disble(run, monkeypatch): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", f"--volume={pathlib.Path.cwd()}://workspace", "--env", @@ -171,6 +177,7 @@ def test_execute_main_stata_license(run, monkeypatch, no_user): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", f"--volume={pathlib.Path.cwd()}://workspace", "--env", diff --git a/tests/test_jupyter.py b/tests/test_jupyter.py index b587368..fa6b180 100644 --- a/tests/test_jupyter.py +++ b/tests/test_jupyter.py @@ -12,6 +12,7 @@ def test_jupyter(run, no_user): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", f"--volume={pathlib.Path.cwd()}://workspace", "-p=1234:1234", diff --git a/tests/test_rstudio.py b/tests/test_rstudio.py new file mode 100644 index 0000000..58b2fbf --- /dev/null +++ b/tests/test_rstudio.py @@ -0,0 +1,57 @@ +import os +import pathlib +from sys import platform + +import pytest + +from opensafely import rstudio +from tests.conftest import run_main + + +@pytest.mark.parametrize("gitconfig_exists", [True, False]) +def test_rstudio(run, tmp_path, monkeypatch, gitconfig_exists): + + home = tmp_path / "home" + home.mkdir() + # linux/macos + monkeypatch.setitem(os.environ, "HOME", str(home)) + # windows + monkeypatch.setitem(os.environ, "USERPROFILE", str(home)) + + if gitconfig_exists: + (home / ".gitconfig").touch() + + if platform == "linux": + uid = os.getuid() + else: + uid = None + + run.expect(["docker", "image", "inspect", "ghcr.io/opensafely-core/rstudio:latest"]) + + expected = [ + "docker", + "run", + "--rm", + "--init", + "--label=opensafely", + "--platform=linux/amd64", + "--interactive", + "--user=0:0", + f"--volume={pathlib.Path.cwd()}://workspace", + "-p=8787:8787", + "--name=test_rstudio", + "--hostname=test_rstudio", + "--env=HOSTPLATFORM=" + platform, + f"--env=HOSTUID={uid}", + ] + + if gitconfig_exists: + expected.append( + "--volume=" + + os.path.join(os.path.expanduser("~"), ".gitconfig") + + ":/home/rstudio/local-gitconfig", + ) + + run.expect(expected + ["ghcr.io/opensafely-core/rstudio"]) + + assert run_main(rstudio, "--port 8787 --name test_rstudio") == 0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 78a4845..4c3ee9d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -63,6 +63,7 @@ def test_run_docker_user_default(run, monkeypatch): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", f"--volume={pathlib.Path.cwd()}://workspace", "ghcr.io/opensafely-core/ehrql:v1", ], @@ -80,6 +81,7 @@ def test_run_docker_user_linux(run, monkeypatch): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--user=uid:gid", f"--volume={pathlib.Path.cwd()}://workspace", "ghcr.io/opensafely-core/ehrql:v1", @@ -96,6 +98,7 @@ def test_run_docker_interactive(run, no_user): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", f"--volume={pathlib.Path.cwd()}://workspace", "ghcr.io/opensafely-core/ehrql:v1", @@ -114,6 +117,7 @@ def test_run_docker_interactive_tty(run, no_user, monkeypatch): "--rm", "--init", "--label=opensafely", + "--platform=linux/amd64", "--interactive", "--tty", f"--volume={pathlib.Path.cwd()}://workspace",