From fd5d674f69dd4e73a7878898182a6093c987736b Mon Sep 17 00:00:00 2001 From: xieshuaihu Date: Wed, 25 Oct 2023 18:39:36 +0800 Subject: [PATCH 01/12] add grpcio grpcio_status to support spark connect --- images/pyspark-notebook/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile index c50701ef58..c2dce68cbe 100644 --- a/images/pyspark-notebook/Dockerfile +++ b/images/pyspark-notebook/Dockerfile @@ -75,7 +75,9 @@ USER ${NB_UID} # 2. Find the pandas version in the file spark/dev/infra/Dockerfile. RUN mamba install --yes \ 'pandas=2.0.3' \ - 'pyarrow' && \ + 'pyarrow' \ + 'grpcio' \ + 'grpcio_status' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" From 003911767494f2d2bdef274efbdd5b948092f214 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 25 Oct 2023 13:03:22 +0200 Subject: [PATCH 02/12] Sort install list --- images/pyspark-notebook/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile index c2dce68cbe..4bbc224131 100644 --- a/images/pyspark-notebook/Dockerfile +++ b/images/pyspark-notebook/Dockerfile @@ -74,10 +74,10 @@ USER ${NB_UID} # 1. Check out the Spark branch you are on. # 2. Find the pandas version in the file spark/dev/infra/Dockerfile. RUN mamba install --yes \ - 'pandas=2.0.3' \ - 'pyarrow' \ + 'grpcio_status' \ 'grpcio' \ - 'grpcio_status' && \ + 'pandas=2.0.3' \ + 'pyarrow' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" From 55ea86d3f508a6ca86b0fb881cb3492f4091af6d Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 25 Oct 2023 13:08:27 +0200 Subject: [PATCH 03/12] Fix package name --- images/pyspark-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile index 4bbc224131..28d40dc2d0 100644 --- a/images/pyspark-notebook/Dockerfile +++ b/images/pyspark-notebook/Dockerfile @@ -74,7 +74,7 @@ USER ${NB_UID} # 1. Check out the Spark branch you are on. # 2. Find the pandas version in the file spark/dev/infra/Dockerfile. RUN mamba install --yes \ - 'grpcio_status' \ + 'grpcio-status' \ 'grpcio' \ 'pandas=2.0.3' \ 'pyarrow' && \ From 8d55c39ff6f3afa8c255d5e248b8b37ed9c68d6a Mon Sep 17 00:00:00 2001 From: xieshuaihu Date: Thu, 26 Oct 2023 09:21:18 +0800 Subject: [PATCH 04/12] Update pyspark docs with new deps grpcio and grpcio-status --- docs/using/selecting.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/using/selecting.md b/docs/using/selecting.md index f85ab4748c..e898c99e2e 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -210,6 +210,8 @@ It contains: - Everything in `jupyter/scipy-notebook` and its ancestor images - [Apache Spark](https://spark.apache.org/) with Hadoop binaries - [pyarrow](https://arrow.apache.org/docs/python/) library +- [grpcio](https://grpc.io/docs/languages/python/quickstart/) +- [grpcio-status](https://github.com/grpc/grpc/tree/master/src/python/grpcio_status) ### jupyter/all-spark-notebook From ba8699815059c0313ef12d6b75cb6eeab8dd4516 Mon Sep 17 00:00:00 2001 From: xieshuaihu Date: Thu, 26 Oct 2023 19:18:01 +0800 Subject: [PATCH 05/12] set grpcio and grpcio-status version as 1.56 --- images/pyspark-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile index 28d40dc2d0..9d969e412b 100644 --- a/images/pyspark-notebook/Dockerfile +++ b/images/pyspark-notebook/Dockerfile @@ -74,8 +74,8 @@ USER ${NB_UID} # 1. Check out the Spark branch you are on. # 2. Find the pandas version in the file spark/dev/infra/Dockerfile. RUN mamba install --yes \ - 'grpcio-status' \ - 'grpcio' \ + 'grpcio-status=1.56' \ + 'grpcio=1.56' \ 'pandas=2.0.3' \ 'pyarrow' && \ mamba clean --all -f -y && \ From df0464cf4e78c525d23e95ded66907ae2ffc9255 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Thu, 26 Oct 2023 16:24:49 +0300 Subject: [PATCH 06/12] Add navigation_with_keys: False and sort options --- docs/conf.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7ef2b342aa..cacfe3d038 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -37,16 +37,17 @@ html_title = "Docker Stacks documentation" html_logo = "_static/jupyter-logo.svg" html_theme_options = { + "logo": { + "text": html_title, + }, + "navigation_with_keys": False, "path_to_docs": "docs", - "repository_url": "https://github.com/jupyter/docker-stacks", "repository_branch": "main", + "repository_url": "https://github.com/jupyter/docker-stacks", + "use_download_button": True, "use_edit_page_button": True, "use_issues_button": True, "use_repository_button": True, - "use_download_button": True, - "logo": { - "text": html_title, - }, } html_last_updated_fmt = "%Y-%m-%d" From cbcacfff6c2cd653f7506b99d531df36ef152bc3 Mon Sep 17 00:00:00 2001 From: xieshuaihu Date: Thu, 26 Oct 2023 22:27:35 +0800 Subject: [PATCH 07/12] exclude grpcio and grpcio-status in test_packages.py --- tests/base-notebook/test_packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/base-notebook/test_packages.py b/tests/base-notebook/test_packages.py index b56d6120e5..51cb209eee 100644 --- a/tests/base-notebook/test_packages.py +++ b/tests/base-notebook/test_packages.py @@ -77,6 +77,8 @@ "python", "r-irkernel", "unixodbc", + "grpcio", + "grpcio-status", ] From 2e35c52f83ed062052029acb9626f34a4cc19e51 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 27 Oct 2023 13:00:22 +0300 Subject: [PATCH 08/12] Introduce policy for new images/packages (#2016) * Introduce policy for new images/packages * Fix GitHub links * Upadte list --- docs/index.rst | 1 + .../new-images-and-packages-policy.md | 35 +++++++++++++++++++ docs/maintaining/tasks.md | 1 + 3 files changed, 37 insertions(+) create mode 100644 docs/maintaining/new-images-and-packages-policy.md diff --git a/docs/index.rst b/docs/index.rst index 875ea2c57f..896ca0f7da 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Table of Contents :maxdepth: 2 :caption: Maintainer Guide + maintaining/new-images-and-packages-policy maintaining/tasks maintaining/aarch64-runner diff --git a/docs/maintaining/new-images-and-packages-policy.md b/docs/maintaining/new-images-and-packages-policy.md new file mode 100644 index 0000000000..b6e7531472 --- /dev/null +++ b/docs/maintaining/new-images-and-packages-policy.md @@ -0,0 +1,35 @@ +# Policy on adding new images and packages + +There are many things we consider, while adding new images and packages. + +Here is a non exhaustive list of things we do care about: + +1. **Software health**, details, and maintenance status + - reasonable versioning is adopted, and the version is considered to be stable + - has been around for several years + - the package maintains documentation + - a changelog is actively maintained + - a release procedure with helpful automation is established + - multiple people are involved in the maintenance of the project + - provides a `conda-forge` package besides a `pypi` package, where both are kept up to date + - supports both `x86_64` and `aarch64` architectures +2. **Installation consequences** + - GitHub Actions build time + - Image sizes + - All requirements should be installed as well +3. Jupyter Docker Stacks _**image fit**_ + - new package or stack is changing (or inherits from) the most suitable stack +4. **Software impact** for users of docker-stacks images + - How this image can help existing users, or maybe reduce the need to build new images +5. Why it shouldn't just be a documented **recipe** +6. Impact on **security** + - Does the package open additional ports, or add new web endpoints, that could be exploited? + +With all this in mind, we have a voting group, which consists of +[mathbunnyru](https://github.com/mathbunnyru), +[consideRatio](https://github.com/consideRatio), +[yuvipanda](https://github.com/yuvipanda) and +[manics](https://github.com/manics). + +This voting group is responsible for accepting or declining new packages and stacks. +The change is accepted, if there are **at least 2 positive votes**. diff --git a/docs/maintaining/tasks.md b/docs/maintaining/tasks.md index 536601e3a4..c6a577ae87 100644 --- a/docs/maintaining/tasks.md +++ b/docs/maintaining/tasks.md @@ -42,6 +42,7 @@ Pushing the `Run Workflow` button will trigger this process. ```{note} In general, we do not add new core images and ask contributors to either create a [recipe](../using/recipes.md) or [community stack](../contributing/stacks.md). +We have a [policy](./new-images-and-packages-policy.md), which we consider when adding new images or new packages to existing images. ``` You can see an example of adding a new image [here](https://github.com/jupyter/docker-stacks/pull/1936/files). From dedb24a8b6521882dfaefb0824127f675859f599 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 27 Oct 2023 13:01:54 +0300 Subject: [PATCH 09/12] Update selecting.md --- docs/using/selecting.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/using/selecting.md b/docs/using/selecting.md index e898c99e2e..26290c4a34 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -209,9 +209,9 @@ It contains: - Everything in `jupyter/scipy-notebook` and its ancestor images - [Apache Spark](https://spark.apache.org/) with Hadoop binaries -- [pyarrow](https://arrow.apache.org/docs/python/) library -- [grpcio](https://grpc.io/docs/languages/python/quickstart/) - [grpcio-status](https://github.com/grpc/grpc/tree/master/src/python/grpcio_status) +- [grpcio](https://grpc.io/docs/languages/python/quickstart/) +- [pyarrow](https://arrow.apache.org/docs/python/) ### jupyter/all-spark-notebook From df64191291c9ed147dff15f1956240456cdd448e Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 27 Oct 2023 13:02:45 +0300 Subject: [PATCH 10/12] Update test_packages.py --- tests/base-notebook/test_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base-notebook/test_packages.py b/tests/base-notebook/test_packages.py index 51cb209eee..ae4fa3f1b6 100644 --- a/tests/base-notebook/test_packages.py +++ b/tests/base-notebook/test_packages.py @@ -69,6 +69,8 @@ "bzip2", "ca-certificates", "conda-forge::blas[build=openblas]", + "grpcio-status", + "grpcio", "hdf5", "jupyterlab-git", "openssl", @@ -77,8 +79,6 @@ "python", "r-irkernel", "unixodbc", - "grpcio", - "grpcio-status", ] From fb105c935511b137f0eafe0b1339f2906c51d890 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 28 Oct 2023 02:03:17 +0300 Subject: [PATCH 11/12] Fix .readthedocs.yaml style --- .readthedocs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f6a12ce2d5..7cc5786e19 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -24,8 +24,8 @@ sphinx: # Optionally build your docs in additional formats such as PDF and ePub # formats: -# - pdf -# - epub +# - pdf +# - epub # Optional but recommended, declare the Python requirements required # to build your documentation From 8cc75b4abc8a33bef72b3a6ea147f005af361f00 Mon Sep 17 00:00:00 2001 From: xieshuaihu Date: Mon, 30 Oct 2023 14:13:16 +0800 Subject: [PATCH 12/12] Update Dockerfile --- images/pyspark-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile index 9d969e412b..28d40dc2d0 100644 --- a/images/pyspark-notebook/Dockerfile +++ b/images/pyspark-notebook/Dockerfile @@ -74,8 +74,8 @@ USER ${NB_UID} # 1. Check out the Spark branch you are on. # 2. Find the pandas version in the file spark/dev/infra/Dockerfile. RUN mamba install --yes \ - 'grpcio-status=1.56' \ - 'grpcio=1.56' \ + 'grpcio-status' \ + 'grpcio' \ 'pandas=2.0.3' \ 'pyarrow' && \ mamba clean --all -f -y && \