Adopt binderhub (#83)

* Add database * Update config * Add repo provider to interface * Create binderhub build handler * Add pydantic models * Build and delete image work * Start server work * Update dev dependencies * Update UI tests * Add missing deps * Add snapshots * Update snapshots * Split unit test module * Reactivate unit tests * Add tests for binderhub build backend * Add docstring * Update tests * Update README * Make tljh optional * Fix open server URL
plasmabio · May 15, 2024 · 6b144d2 · 6b144d2
1 parent 5b6e3b4
commit 6b144d2
Show file tree

Hide file tree

Showing 80 changed files with 2,008 additions and 234 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches: master
   pull_request:
-    branches: "*"
+    branches: '*'
 
 jobs:
   build:
@@ -70,7 +70,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ['3.8', '3.9', '3.10']
 
     steps:
       - name: Checkout
@@ -91,15 +91,24 @@ jobs:
           python -m pip install tljh_repo2docker*.whl
           npm -g install configurable-http-proxy
 
-      - name: Run Tests
+      - name: Run local build backend tests
+        working-directory: tljh_repo2docker/tests
         run: |
-          python -m pytest --cov
+          python -m pytest local_build   --cov
+
+      - name: Run binderhub build backend tests
+        working-directory: tljh_repo2docker/tests
+        run: |
+          python -m pytest binderhub_build   --cov
 
   integration-tests:
     name: Integration tests
     needs: build
     runs-on: ubuntu-latest
-
+    strategy:
+      fail-fast: false
+      matrix:
+        build-backend: ['local', 'binderhub']
     env:
       PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers
 
@@ -139,16 +148,17 @@ jobs:
         run: npx playwright install chromium
         working-directory: ui-tests
 
-      - name: Execute integration tests
+      - name: Execute integration tests with ${{ matrix.build-backend }} build backend
         working-directory: ui-tests
         run: |
-          npx playwright test
+          npm run test:${{ matrix.build-backend }}
 
       - name: Upload Playwright Test report
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: tljh-playwright-tests
           path: |
-            ui-tests/test-results
+            ui-tests/local-test-results
+            ui-tests/binderhub-test-results
             ui-tests/playwright-report
diff --git a/.gitignore b/.gitignore
@@ -30,3 +30,4 @@ lib/
 
 # Hatch version
 _version.py
+*.sqlite
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -46,10 +46,16 @@ docker pull quay.io/jupyterhub/repo2docker:main
 
 ## Run
 
-Finally, start `jupyterhub` with the config in `debug` mode:
+Finally, start `jupyterhub` with local build backend:
 
 ```bash
-python -m jupyterhub -f jupyterhub_config.py --debug
+python -m jupyterhub -f ui-tests/jupyterhub_config_local.py --debug
+```
+
+or using `binderhub` build backend
+
+```bash
+python -m jupyterhub -f ui-tests/jupyterhub_config_binderhub.py --debug
 ```
 
 Open https://localhost:8000 in a web browser.

diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 ![Github Actions Status](https://github.com/plasmabio/tljh-repo2docker/workflows/Tests/badge.svg)
 
-TLJH plugin providing a JupyterHub service to build and use Docker images as user environments. The Docker images are built using [`repo2docker`](https://repo2docker.readthedocs.io/en/latest/).
+TLJH plugin provides a JupyterHub service to build and use Docker images as user environments. The Docker images can be built locally using [`repo2docker`](https://repo2docker.readthedocs.io/en/latest/) or via the [`binderhub`](https://binderhub.readthedocs.io/en/latest/) service.
 
 ## Requirements
 
@@ -46,8 +46,10 @@ The available settings for this service are:
 - `default_memory_limit`: Default memory limit of a user server; defaults to `None`
 - `default_cpu_limit`: Default CPU limit of a user server; defaults to `None`
 - `machine_profiles`: Instead of entering directly the CPU and Memory value, `tljh-repo2docker` can be configured with pre-defined machine profiles and users can only choose from the available option; defaults to `[]`
+- `binderhub_url`: The optional URL of the `binderhub` service. If it is available, `tljh-repo2docker` will use this service to build images.
+- `db_url`: The connection string of the database. `tljh-repo2docker` needs a database to store the image metadata. By default, it will create a `sqlite` database in the starting directory of the service. To use other databases (`PostgreSQL` or `MySQL`), users need to specify the connection string via this config and install the additional drivers (`asyncpg` or `aiomysql`).
 
-This service requires the following scopes : `read:users`, `admin:servers` and `read:roles:users`. Here is an example of registering `tljh_repo2docker`'s service with JupyterHub
+This service requires the following scopes : `read:users`, `admin:servers` and `read:roles:users`. If `binderhub` service is used, ` access:services!service=binder`is also needed. Here is an example of registering `tljh_repo2docker`'s service with JupyterHub
 
 ```python
 # jupyterhub_config.py
@@ -78,7 +80,12 @@ c.JupyterHub.load_roles = [
     {
         "description": "Role for tljh_repo2docker service",
         "name": "tljh-repo2docker-service",
-        "scopes": ["read:users", "admin:servers", "read:roles:users"],
+        "scopes": [
+            "read:users",
+            "read:roles:users",
+            "admin:servers",
+            "access:services!service=binder",
+        ],
         "services": ["tljh_repo2docker"],
     },
     {
@@ -147,38 +154,46 @@ c.JupyterHub.load_roles = [
 
 The _Environments_ page shows the list of built environments, as well as the ones currently being built:
 
-![environments](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/environment-list-linux.png)
+![environments](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/environment-list.png)
 
 ### Add a new environment
 
 Just like on [Binder](https://mybinder.org), new environments can be added by clicking on the _Add New_ button and providing a URL to the repository. Optional names, memory, and CPU limits can also be set for the environment:
 
-![add-new](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/environment-dialog-linux.png)
+![add-new](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/environment-dialog.png)
+
+> [!NOTE]
+> If the build backend is `binderhub` service, users need to select the [repository provider](https://binderhub.readthedocs.io/en/latest/developer/repoproviders.html) and can not specify the custom build arguments
+
+![add-new-binderhub](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/binderhub_snapshots/ui.test.ts/environment-dialog.png)
 
 ### Follow the build logs
 
 Clicking on the _Logs_ button will open a new dialog with the build logs:
 
-![logs](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/environment-console-linux.png)
+![logs](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/environment-console.png)
 
 ### Select an environment
 
 Once ready, the environments can be selected from the JupyterHub spawn page:
 
-![select-env](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/servers-dialog-linux.png)
+![select-env](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/servers-dialog.png)
 
 ### Private Repositories
 
 `tljh-repo2docker` also supports building environments from private repositories.
 
 It is possible to provide the `username` and `password` in the `Credentials` section of the form:
 
-![image](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/environment-dialog-linux.png)
+![image](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/environment-dialog.png)
 
 On GitHub and GitLab, a user might have to first create an access token with `read` access to use as the password:
 
 ![image](https://user-images.githubusercontent.com/591645/107350843-39c3bf80-6aca-11eb-8b82-6fa95ba4c7e4.png)
 
+> [!NOTE]
+> The `binderhub` build backend does not support configuring private repositories credentials from the interface.
+
 ### Machine profiles
 
 Instead of entering directly the CPU and Memory value, `tljh-repo2docker` can be configured with pre-defined machine profiles and users can only choose from the available options. The following configuration will add 3 machines with labels Small, Medium and Large to the profile list:

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,5 +1,7 @@
 git+https://github.com/jupyterhub/[email protected]
+git+https://github.com/jupyterhub/binderhub.git@main
 jupyterhub>=4,<5
+alembic>=1.13.0,<1.14
 pytest
 pytest-aiohttp
 pytest-asyncio

diff --git a/pyproject.toml b/pyproject.toml
@@ -7,13 +7,21 @@ dependencies = [
   "aiodocker~=0.19",
   "dockerspawner~=12.1",
   "jupyter_client>=6.1,<8",
-  "httpx"
+  "httpx",
+  "sqlalchemy>=2",
+  "pydantic>=2,<3",
+  "alembic>=1.13,<2",
+  "jupyter-repo2docker>=2024,<2025",
+  "aiosqlite~=0.19.0"
 ]
 dynamic = ["version"]
 license = {file = "LICENSE"}
 name = "tljh-repo2docker"
 readme = "README.md"
 
+[project.scripts]
+tljh_repo2docker_upgrade_db = "tljh_repo2docker.dbutil:main"
+
 [project.entry-points.tljh]
 tljh_repo2docker = "tljh_repo2docker"
 
@@ -56,10 +64,7 @@ source_dir = "src"
 version_cmd = "hatch version"
 
 [tool.jupyter-releaser.hooks]
-before-build-npm = [
-  "npm install",
-  "npm run build:prod",
-]
+before-build-npm = ["npm install", "npm run build:prod"]
 before-build-python = ["npm run clean"]
 before-bump-version = ["python -m pip install hatch"]
 

diff --git a/src/environments/App.tsx b/src/environments/App.tsx
@@ -16,6 +16,8 @@ export interface IAppProps {
   default_cpu_limit: string;
   default_mem_limit: string;
   machine_profiles: IMachineProfile[];
+  use_binderhub: boolean;
+  repo_providers?: { label: string; value: string }[];
 }
 export default function App(props: IAppProps) {
   const jhData = useJupyterhub();
@@ -35,6 +37,8 @@ export default function App(props: IAppProps) {
               default_cpu_limit={props.default_cpu_limit}
               default_mem_limit={props.default_mem_limit}
               machine_profiles={props.machine_profiles}
+              use_binderhub={props.use_binderhub}
+              repo_providers={props.repo_providers}
             />
             <EnvironmentList {...props} />
           </Stack>

diff --git a/src/environments/EnvironmentList.tsx b/src/environments/EnvironmentList.tsx
@@ -59,7 +59,7 @@ const columns: GridColDef[] = [
       ) : params.value === 'building' ? (
         <EnvironmentLogButton
           name={params.row.display_name}
-          image={params.row.image_name}
+          image={params.row.uid ?? params.row.image_name}
         />
       ) : null;
     }
@@ -75,7 +75,7 @@ const columns: GridColDef[] = [
       return (
         <RemoveEnvironmentButton
           name={params.row.display_name}
-          image={params.row.image_name}
+          image={params.row.uid ?? params.row.image_name}
         />
       );
     }

diff --git a/src/environments/LogDialog.tsx b/src/environments/LogDialog.tsx
@@ -65,13 +65,13 @@ function _EnvironmentLogButton(props: IEnvironmentLogButton) {
       eventSource.onmessage = event => {
         const data = JSON.parse(event.data);
 
+        terminal.write(data.message);
+        fitAddon.fit();
         if (data.phase === 'built') {
           eventSource.close();
           setBuilt(true);
           return;
         }
-        terminal.write(data.message);
-        fitAddon.fit();
       };
     }
   }, [jhData, props.image]);
Original file line number	Diff line number	Diff line change
Expand Up		@@ -30,3 +30,4 @@ lib/

		# Hatch version
		_version.py
		*.sqlite