#233: Add script to generate package diffs between releases (#205)

- Add script to generate package diffs between releases - Unify package list names and formats - Replace manual apt-get installation with install script in python-3.6-data-science-cuda-EXASOL-6.2.0 flavor, except for cuda_deps Related to exasol/script-languages-release#233
exasol · Aug 23, 2021 · 828fd3e · 828fd3e
1 parent 100fc4f
commit 828fd3e
Show file tree

Hide file tree

Showing 20 changed files with 597 additions and 87 deletions.
diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md b/flavors/python-3.6-data-science-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md
@@ -8,7 +8,7 @@
 - [Language dependencies](flavor_base/language_deps/packages/apt_get_packages)
 - Flavor packages
   - [Ubuntu packages](flavor_base/flavor_base_deps/packages/apt_get_packages)
-  - [Python3 pip packages](flavor_base/flavor_base_deps/packages/pip3_packages)
+  - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages)
 - Customization
   - [Ubuntu packages](flavor_customization/packages/apt_get_packages)
   - [Python3 pip packages](flavor_customization/packages/python3_pip_packages)
diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile
@@ -1,8 +1,9 @@
 FROM {{ language_deps }}
 
-RUN mkdir -p /build_info/packages
-COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps
+RUN mkdir -p /build_info/packages/flavor_base_deps
 
-RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages
+COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions --allow-no-version
 
-RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/pip3_packages --python-binary python3
+COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps
+RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version
diff --git a/.../python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages b/.../python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages
diff --git a/...-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages b/...-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages
@@ -0,0 +1,19 @@
+pyexasol|
+keras|2.3.1
+tensorflow|1.13.1
+tensorflow-hub|0.4.0
+kmodes|
+seaborn|
+matplotlib|
+imbalanced-learn|
+lifelines|
+nltk|
+gensim|
+lxml|
+ujson|
+numpy|
+scipy|1.2.1 # due to https://github.com/CamDavidsonPilon/lifelines/issues/725
+scikit-learn|
+pycurl|
+statsmodels|
+PyYAML|
diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile
@@ -1,14 +1,14 @@
 FROM {{udfclient_deps}}
 
 RUN mkdir -p /build_info/packages/language_deps
-COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps
 
-RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages
+COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions --allow-no-version
 
 RUN /scripts/install_scripts/install_python3.6_pip.sh
 
-COPY language_deps/packages/pip3_packages /build_info/packages/language_deps
-RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/pip3_packages --python-binary python3
+COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps
+RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version
 
 ENV PYTHON3_PREFIX /usr
 ENV PYTHON3_VERSION python3.6

diff --git a/...base/language_deps/packages/pip3_packages → ...nguage_deps/packages/python3_pip_packages b/...base/language_deps/packages/pip3_packages → ...nguage_deps/packages/python3_pip_packages
diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_customization/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_customization/Dockerfile
@@ -72,6 +72,6 @@ RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flav
 # if you want to copy files or directories located in the same directory as this Dockerfile
 # -----------------------------------------------------------------------------------------------------
 
-# COPY flavor_customization/packages/pip3_packages /build_info/packages/flavor_base_deps
+# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization
 # ADD flavor_customization/code.tar.gz /code
 
diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md
@@ -8,7 +8,7 @@
 - [Language dependencies](flavor_base/language_deps/packages/apt_get_packages)
 - Flavor packages
   - [Ubuntu packages](flavor_base/flavor_base_deps/packages/apt_get_packages)
-  - [Python3 pip packages](flavor_base/flavor_base_deps/packages/pip3_packages)
+  - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages)
 - Customization
   - [Ubuntu packages](flavor_customization/packages/apt_get_packages)
   - [Python3 pip packages](flavor_customization/packages/python3_pip_packages)
diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py
@@ -21,7 +21,8 @@ def get_build_step(self) -> str:
         return "udfclient_deps"
 
     def get_additional_build_directories_mapping(self) -> Dict[str, str]:
-        return {"01_nodoc": "ext/01_nodoc"}
+        return {"01_nodoc": "ext/01_nodoc",
+                "scripts": "ext/scripts"}
 
     def requires_tasks(self):
         return {"cuda_deps": AnalyzeCudaDeps}

diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile
@@ -1,14 +1,9 @@
 FROM {{ language_deps }}
 
-RUN mkdir -p /build_info/packages
-COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps
+RUN mkdir -p /build_info/packages/flavor_base_deps
 
-RUN apt-get -y update && \
-    apt-get -y install $(cat /build_info/packages/flavor_base_deps/apt_get_packages) && \
-	locale-gen en_US.UTF-8 && \
-    update-locale LC_ALL=en_US.UTF-8 && \
-    apt-get -y clean && \
-    apt-get -y autoremove && \
-    ldconfig
+COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions --allow-no-version
 
-RUN /scripts/install_scripts/install_packages_batch.sh "PYTHON3-PIP" "pip install --ignore-installed <<list>>" "'<<package>>'" " " /build_info/packages/flavor_base_deps/pip3_packages
+COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps
+RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version
diff --git a/...on-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages b/...on-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages
diff --git a/...data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages b/...data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages
@@ -0,0 +1,19 @@
+pyexasol
+keras|2.3.1
+tensorflow-gpu|1.13.1 # due to the dependency of tensorflow's pip package to the fixed CUDA Driver Version 410.104
+tensorflow-hub|0.4.0
+kmodes
+seaborn
+matplotlib
+imbalanced-learn
+lifelines
+nltk
+gensim
+lxml
+ujson
+numpy
+scipy|1.2.1 # due to https://github.com/CamDavidsonPilon/lifelines/issues/725
+scikit-learn
+pycurl
+statsmodels
+PyYAML
diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile
@@ -1,26 +1,14 @@
 FROM {{udfclient_deps}}
 
 RUN mkdir -p /build_info/packages/language_deps
-COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps
 
-# ppa:deadsnakes/ppa needed for python3.6 under ubuntu:16.04
-RUN apt-get update && \
-    apt-get -y install software-properties-common && \
-    add-apt-repository ppa:deadsnakes/ppa && \
-    apt-get -y update && \
-    apt-get -y install $(cat /build_info/packages/language_deps/apt_get_packages) && \
-    locale-gen en_US.UTF-8 && \
-    update-locale LC_ALL=en_US.UTF-8 && \
-    apt-get -y clean && \
-    apt-get -y autoremove && \
-    ldconfig
+COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions --allow-no-version
 
-COPY scripts /scripts
 RUN /scripts/install_scripts/install_python3.6_pip.sh
 
-COPY scripts /scripts
-COPY language_deps/packages/pip3_packages /build_info/packages/language_deps
-RUN /scripts/install_scripts/install_packages_batch.sh "PYTHON3-PIP" "pip install --ignore-installed <<list>>" "<<package>>" " " /build_info/packages/language_deps/pip3_packages
+COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps
+RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version
 
 ENV PYTHON3_PREFIX /usr
 ENV PYTHON3_VERSION python3.6

diff --git a/...base/language_deps/packages/pip3_packages → ...nguage_deps/packages/python3_pip_packages b/...base/language_deps/packages/pip3_packages → ...nguage_deps/packages/python3_pip_packages
diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/udfclient_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/udfclient_deps/Dockerfile
@@ -6,18 +6,12 @@ RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNT
 
 COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
 
+COPY scripts /scripts
+
 RUN mkdir -p /build_info/packages
 COPY udfclient_deps/packages /build_info/packages/udfclient_deps
 
-RUN apt-get update
-
-RUN apt-get update && \
-    apt-get -y install $(cat /build_info/packages/udfclient_deps/apt_get_packages) && \
-    locale-gen en_US.UTF-8 && \
-    update-locale LC_ALL=en_US.UTF-8 && \
-    apt-get -y clean && \
-    apt-get -y autoremove && \
-    ldconfig
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages
 
 RUN addgroup --gid 1000 exasolution
 RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone"

diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_customization/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_customization/Dockerfile
@@ -72,6 +72,6 @@ RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flav
 # if you want to copy files or directories located in the same directory as this Dockerfile
 # -----------------------------------------------------------------------------------------------------
 
-# COPY flavor_customization/packages/pip3_packages /build_info/packages/flavor_base_deps
+# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization
 # ADD flavor_customization/code.tar.gz /code
 
diff --git a/flavors/python-3.6-minimal-EXASOL-6.2.0/flavor_customization/Dockerfile b/flavors/python-3.6-minimal-EXASOL-6.2.0/flavor_customization/Dockerfile
@@ -72,6 +72,6 @@ RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flav
 # if you want to copy files or directories located in the same directory as this Dockerfile
 # -----------------------------------------------------------------------------------------------------
 
-# COPY flavor_customization/packages/pip3_packages /build_info/packages/flavor_base_deps
+# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization
 # ADD flavor_customization/code.tar.gz /code