From 0b207a7f0c97e39f07f5e8b26d66555a72f3f9fe Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 19 Aug 2021 16:19:08 +0200 Subject: [PATCH 01/17] WIP: Add Package Diff script --- scripts/generate_package_diffs_for_flavors.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 scripts/generate_package_diffs_for_flavors.py diff --git a/scripts/generate_package_diffs_for_flavors.py b/scripts/generate_package_diffs_for_flavors.py new file mode 100644 index 000000000..9ad9aa403 --- /dev/null +++ b/scripts/generate_package_diffs_for_flavors.py @@ -0,0 +1,78 @@ +import sys +import pandas as pd +import subprocess +from pathlib import Path +from tempfile import TemporaryDirectory +from io import StringIO + + +def parse_package_list_file(file_path:Path)->str: + parse_package_list_file_command = ["perl","ext/scripts/list_newest_versions/extract_columns_from_package_lisl.pl","--file",str(file_path),"--columns","0,1"] + result = subprocess.run(parse_package_list_file_command,stdout=subprocess.PIPE) + result.check_returncode() + result_string = result.stdout.decode("UTF-8") + return result_string + +def compare_package_lists(package_list_1:str, package_list_2:str)->pd.DataFrame: + package_list_1_df = pd.read_csv(StringIO(package_list_1), delimiter="|", names=["package","version"]) + package_list_2_df = pd.read_csv(StringIO(package_list_2), delimiter="|", names=["package","version"]) + diff_df = pd.merge(package_list_1_df, package_list_2_df, how='outer', on='package',sort=False) + diff_df = diff_df.sort_values("package") + new = diff_df["version_y"].isnull() + removed = diff_df["version_x"].isnull() + updated = ~diff_df["version_x"].isnull() & (diff_df["version_x"]!=diff_df["version_y"]) + diff_df["status"]="" +# diff_df["status"].loc[new]="NEW" +# diff_df["status",removed]="REMOVED" + diff_df["status"].values[updated]="UPDATED" + diff_df = diff_df.reset_index(drop=True) + return diff_df + +def compare_build_step(flavor_path:Path, build_step_path:Path, working_copy_1:Path, working_copy_2:Path): + packages_path = Path(build_step_path,"packages") + if packages_path.is_dir(): + for package_list_file in packages_path.iterdir(): + package_list_working_copy_str_1 = parse_package_list_file(Path(working_copy_1,package_list_file)) + package_list_working_copy_str_2 = parse_package_list_file(Path(working_copy_2,package_list_file)) + diff = compare_package_lists(package_list_working_copy_str_1, package_list_working_copy_str_2) + print(diff) + + +def compare_flavor(flavor_path:Path, working_copy_1:Path, working_copy_2:Path): + flavor_base_path = Path(flavor_path,"flavor_base") + if flavor_base_path.is_dir(): + for build_step_path in flavor_base_path.iterdir(): + if build_step_path.is_dir(): + compare_build_step(flavor_path,build_step_path,working_copy_1,working_copy_2) + + +get_last_tag_command = ["git","describe","--abbrev=0","--tags"] +last_tag_result = subprocess.run(get_last_tag_command,stdout=subprocess.PIPE) +last_tag_result.check_returncode() +last_tag = last_tag_result.stdout.decode("UTF-8").strip() +print(last_tag) + +with TemporaryDirectory() as tmp_dir: + print(tmp_dir) + checkout_last_tag_command = ["git","worktree","add",tmp_dir,last_tag] + checkout_last_tag_result = subprocess.run(checkout_last_tag_command,stderr=subprocess.PIPE) + checkout_last_tag_result.check_returncode() + init_submodule_command = ["git","submodule","update","--init"] + init_submodule_result = subprocess.run(init_submodule_command,cwd=tmp_dir,stderr=subprocess.PIPE) + init_submodule_result.check_returncode() + working_copy_root = Path(".") + for flavor_path in Path("flavors").iterdir(): + if flavor_path.is_dir(): + compare_flavor(flavor_path.relative_to(working_copy_root),working_copy_root,tmp_dir) + + +# package_list_1_path = sys.argv[1] +# package_list_2_path = sys.argv[2] +# package_list_1 = pd.read_csv(package_list_1_path, delimiter="|", names=["package","version"]) +# package_list_2 = pd.read_csv(package_list_2_path, delimiter="|", names=["package","version"]) +# diff = pd.merge(package_list_1, package_list_2, how='outer', on='package',sort=False) +# diff = diff.sort_values("package") +# diff["Updated"] = diff["version_x"]!=diff["version_y"] +# diff = diff.reset_index(drop=True) +# diff.to_markdown(sys.stdout) +# print() From bd05d600fe656d2080047e6c001c939971d1ffa9 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 20 Aug 2021 16:47:57 +0200 Subject: [PATCH 02/17] Unify the package list names and format --- .../FLAVOR_DESCRIPTION.md | 2 +- .../flavor_base/flavor_base_deps/Dockerfile | 14 +++++++++++--- .../flavor_base_deps/packages/pip3_packages | 19 ------------------- .../packages/python3_pip_packages | 19 +++++++++++++++++++ .../flavor_base/language_deps/Dockerfile | 18 +++++++++++++++--- .../{pip3_packages => python3_pip_packages} | 0 .../flavor_customization/Dockerfile | 2 +- .../FLAVOR_DESCRIPTION.md | 2 +- .../flavor_base/flavor_base_deps/Dockerfile | 6 ++++-- .../flavor_base_deps/packages/pip3_packages | 19 ------------------- .../packages/python3_pip_packages | 19 +++++++++++++++++++ .../flavor_base/language_deps/Dockerfile | 4 ++-- .../{pip3_packages => python3_pip_packages} | 0 .../flavor_customization/Dockerfile | 2 +- .../flavor_customization/Dockerfile | 2 +- 15 files changed, 75 insertions(+), 53 deletions(-) delete mode 100644 flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages create mode 100644 flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages rename flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/packages/{pip3_packages => python3_pip_packages} (100%) delete mode 100644 flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages create mode 100644 flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages rename flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/packages/{pip3_packages => python3_pip_packages} (100%) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md b/flavors/python-3.6-data-science-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md index 8cb82693f..a9a54acf9 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md @@ -8,7 +8,7 @@ - [Language dependencies](flavor_base/language_deps/packages/apt_get_packages) - Flavor packages - [Ubuntu packages](flavor_base/flavor_base_deps/packages/apt_get_packages) - - [Python3 pip packages](flavor_base/flavor_base_deps/packages/pip3_packages) + - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages) - Customization - [Ubuntu packages](flavor_customization/packages/apt_get_packages) - [Python3 pip packages](flavor_customization/packages/python3_pip_packages) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index a3f0953d4..d1720eeed 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -1,8 +1,16 @@ FROM {{ language_deps }} RUN mkdir -p /build_info/packages -COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps +COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages +RUN apt-get -y update && \ + apt-get -y install $(cat /build_info/packages/flavor_base_deps/apt_get_packages) && \ + locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + apt-get -y clean && \ + apt-get -y autoremove && \ + ldconfig -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/pip3_packages --python-binary python3 + +COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packeges --python-binary python3 diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages deleted file mode 100644 index ce30344cd..000000000 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages +++ /dev/null @@ -1,19 +0,0 @@ -pyexasol -keras==2.3.1 -tensorflow==1.13.1 -tensorflow-hub==0.4.0 -kmodes -seaborn -matplotlib -imbalanced-learn -lifelines -nltk -gensim -lxml -ujson -numpy -scipy<=1.2.1 # due to https://github.com/CamDavidsonPilon/lifelines/issues/725 -scikit-learn -pycurl -statsmodels -PyYAML diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages new file mode 100644 index 000000000..e0423026d --- /dev/null +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages @@ -0,0 +1,19 @@ +pyexasol| +keras|2.3.1 +tensorflow|1.13.1 +tensorflow-hub|0.4.0 +kmodes| +seaborn| +matplotlib| +imbalanced-learn| +lifelines| +nltk| +gensim| +lxml| +ujson| +numpy| +scipy|1.2.1 # due to https://github.com/CamDavidsonPilon/lifelines/issues/725 +scikit-learn| +pycurl| +statsmodels| +PyYAML| diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index 4d578d745..e238defab 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -3,12 +3,24 @@ FROM {{udfclient_deps}} RUN mkdir -p /build_info/packages/language_deps COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages +# ppa:deadsnakes/ppa needed for python3.6 under ubuntu:16.04 +RUN apt-get update && \ + apt-get -y install software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt-get -y update && \ + apt-get -y install $(cat /build_info/packages/language_deps/apt_get_packages) && \ + locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + apt-get -y clean && \ + apt-get -y autoremove && \ + ldconfig +COPY scripts /scripts RUN /scripts/install_scripts/install_python3.6_pip.sh -COPY language_deps/packages/pip3_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/pip3_packages --python-binary python3 +COPY scripts /scripts +COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packges --python-binary python3 ENV PYTHON3_PREFIX /usr ENV PYTHON3_VERSION python3.6 diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/packages/pip3_packages b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/packages/python3_pip_packages similarity index 100% rename from flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/packages/pip3_packages rename to flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/packages/python3_pip_packages diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_customization/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_customization/Dockerfile index acff6f994..c4f16a063 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_customization/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_customization/Dockerfile @@ -72,6 +72,6 @@ RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flav # if you want to copy files or directories located in the same directory as this Dockerfile # ----------------------------------------------------------------------------------------------------- -# COPY flavor_customization/packages/pip3_packages /build_info/packages/flavor_base_deps +# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization # ADD flavor_customization/code.tar.gz /code diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md index 03cb8d374..f73f5b0d3 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/FLAVOR_DESCRIPTION.md @@ -8,7 +8,7 @@ - [Language dependencies](flavor_base/language_deps/packages/apt_get_packages) - Flavor packages - [Ubuntu packages](flavor_base/flavor_base_deps/packages/apt_get_packages) - - [Python3 pip packages](flavor_base/flavor_base_deps/packages/pip3_packages) + - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages) - Customization - [Ubuntu packages](flavor_customization/packages/apt_get_packages) - [Python3 pip packages](flavor_customization/packages/python3_pip_packages) diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index adfd3ad6a..d1720eeed 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -1,7 +1,7 @@ FROM {{ language_deps }} RUN mkdir -p /build_info/packages -COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps +COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps RUN apt-get -y update && \ apt-get -y install $(cat /build_info/packages/flavor_base_deps/apt_get_packages) && \ @@ -11,4 +11,6 @@ RUN apt-get -y update && \ apt-get -y autoremove && \ ldconfig -RUN /scripts/install_scripts/install_packages_batch.sh "PYTHON3-PIP" "pip install --ignore-installed <>" "'<>'" " " /build_info/packages/flavor_base_deps/pip3_packages \ No newline at end of file + +COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packeges --python-binary python3 diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages deleted file mode 100644 index 408a01c09..000000000 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/pip3_packages +++ /dev/null @@ -1,19 +0,0 @@ -pyexasol -keras==2.3.1 -tensorflow-gpu==1.13.1 # due to the dependency of tensorflow's pip package to the fixed CUDA Driver Version 410.104 -tensorflow-hub==0.4.0 -kmodes -seaborn -matplotlib -imbalanced-learn -lifelines -nltk -gensim -lxml -ujson -numpy -scipy<=1.2.1 # due to https://github.com/CamDavidsonPilon/lifelines/issues/725 -scikit-learn -pycurl -statsmodels -PyYAML diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages new file mode 100644 index 000000000..2efdf1b38 --- /dev/null +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/python3_pip_packages @@ -0,0 +1,19 @@ +pyexasol +keras|2.3.1 +tensorflow-gpu|1.13.1 # due to the dependency of tensorflow's pip package to the fixed CUDA Driver Version 410.104 +tensorflow-hub|0.4.0 +kmodes +seaborn +matplotlib +imbalanced-learn +lifelines +nltk +gensim +lxml +ujson +numpy +scipy|1.2.1 # due to https://github.com/CamDavidsonPilon/lifelines/issues/725 +scikit-learn +pycurl +statsmodels +PyYAML diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index f216495b6..e238defab 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -19,8 +19,8 @@ COPY scripts /scripts RUN /scripts/install_scripts/install_python3.6_pip.sh COPY scripts /scripts -COPY language_deps/packages/pip3_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_packages_batch.sh "PYTHON3-PIP" "pip install --ignore-installed <>" "<>" " " /build_info/packages/language_deps/pip3_packages +COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packges --python-binary python3 ENV PYTHON3_PREFIX /usr ENV PYTHON3_VERSION python3.6 diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/packages/pip3_packages b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/packages/python3_pip_packages similarity index 100% rename from flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/packages/pip3_packages rename to flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/packages/python3_pip_packages diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_customization/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_customization/Dockerfile index acff6f994..c4f16a063 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_customization/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_customization/Dockerfile @@ -72,6 +72,6 @@ RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flav # if you want to copy files or directories located in the same directory as this Dockerfile # ----------------------------------------------------------------------------------------------------- -# COPY flavor_customization/packages/pip3_packages /build_info/packages/flavor_base_deps +# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization # ADD flavor_customization/code.tar.gz /code diff --git a/flavors/python-3.6-minimal-EXASOL-6.2.0/flavor_customization/Dockerfile b/flavors/python-3.6-minimal-EXASOL-6.2.0/flavor_customization/Dockerfile index acff6f994..c4f16a063 100644 --- a/flavors/python-3.6-minimal-EXASOL-6.2.0/flavor_customization/Dockerfile +++ b/flavors/python-3.6-minimal-EXASOL-6.2.0/flavor_customization/Dockerfile @@ -72,6 +72,6 @@ RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flav # if you want to copy files or directories located in the same directory as this Dockerfile # ----------------------------------------------------------------------------------------------------- -# COPY flavor_customization/packages/pip3_packages /build_info/packages/flavor_base_deps +# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization # ADD flavor_customization/code.tar.gz /code From cdb1b7c555f0e5b9d3ccc7ef4f30f63d7e77c649 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 20 Aug 2021 16:49:02 +0200 Subject: [PATCH 03/17] WIP: generate_package_diffs_for_flavors.py --- scripts/generate_package_diffs_for_flavors.py | 152 +++++++++++++----- 1 file changed, 112 insertions(+), 40 deletions(-) diff --git a/scripts/generate_package_diffs_for_flavors.py b/scripts/generate_package_diffs_for_flavors.py index 9ad9aa403..1f29f6d12 100644 --- a/scripts/generate_package_diffs_for_flavors.py +++ b/scripts/generate_package_diffs_for_flavors.py @@ -1,78 +1,150 @@ import sys import pandas as pd +import numpy as np import subprocess from pathlib import Path from tempfile import TemporaryDirectory from io import StringIO - +import re +from typing import Dict, List +import tempfile def parse_package_list_file(file_path:Path)->str: - parse_package_list_file_command = ["perl","ext/scripts/list_newest_versions/extract_columns_from_package_lisl.pl","--file",str(file_path),"--columns","0,1"] + parse_package_list_file_command = [ + "perl","ext/scripts/list_newest_versions/extract_columns_from_package_lisl.pl", + "--file",str(file_path),"--columns","0,1"] result = subprocess.run(parse_package_list_file_command,stdout=subprocess.PIPE) result.check_returncode() result_string = result.stdout.decode("UTF-8") return result_string def compare_package_lists(package_list_1:str, package_list_2:str)->pd.DataFrame: - package_list_1_df = pd.read_csv(StringIO(package_list_1), delimiter="|", names=["package","version"]) - package_list_2_df = pd.read_csv(StringIO(package_list_2), delimiter="|", names=["package","version"]) - diff_df = pd.merge(package_list_1_df, package_list_2_df, how='outer', on='package',sort=False) - diff_df = diff_df.sort_values("package") - new = diff_df["version_y"].isnull() - removed = diff_df["version_x"].isnull() - updated = ~diff_df["version_x"].isnull() & (diff_df["version_x"]!=diff_df["version_y"]) - diff_df["status"]="" -# diff_df["status"].loc[new]="NEW" -# diff_df["status",removed]="REMOVED" - diff_df["status"].values[updated]="UPDATED" + package_list_1_df = pd.read_csv(StringIO(package_list_1), delimiter="|", names=["Package","Version1"]) + package_list_1_df["Version1"]=package_list_1_df["Version1"].replace("<<<<1>>>>","No version specified") + package_list_2_df = pd.read_csv(StringIO(package_list_2), delimiter="|", names=["Package","Version2"]) + package_list_2_df["Version2"]=package_list_2_df["Version2"].replace("<<<<1>>>>","No version specified") + diff_df = pd.merge(package_list_1_df, package_list_2_df, how='outer', on='Package',sort=False) + new = diff_df["Version1"].isnull() & ~diff_df["Version2"].isnull() + removed = diff_df["Version2"].isnull() & ~diff_df["Version1"].isnull() + updated = ~diff_df["Version1"].isnull() & ~diff_df["Version2"].isnull() & (diff_df["Version1"]!=diff_df["Version2"]) + diff_df["Status"]="" + diff_df["Status"].values[new]="NEW" + diff_df["Status"].values[removed]="REMOVED" + diff_df["Status"].values[updated]="UPDATED" + diff_df = diff_df.fillna("") + diff_df = diff_df.sort_values("Status",ascending=False) diff_df = diff_df.reset_index(drop=True) return diff_df -def compare_build_step(flavor_path:Path, build_step_path:Path, working_copy_1:Path, working_copy_2:Path): +def convert_requirements_file(package_list_str): + def convert_line(line): + line = line.replace("|<<<<1>>>>","|") + line = line.replace("==","|") + line = line.replace(">=","|") + line = line.replace("<=","|") + line = re.sub(r"\|$","",line) + if not "|" in line: + line += "|<<<<1>>>>" + return line + result = "\n".join(convert_line(line) for line in package_list_str.splitlines()) + return result + +def load_package_file_or_alternative(working_copy:Path, package_list_file:Path, package_list_file_name_alternatives:Dict[str,List[str]]): + package_list_str = "" + possible_package_list_file_names = [package_list_file.name] + if package_list_file.name in package_list_file_name_alternatives: + possible_package_list_file_names += package_list_file_name_alternatives[package_list_file.name] + for package_list_file_name in possible_package_list_file_names: + package_list_file = Path(package_list_file.parent, package_list_file_name) + package_list_file_path = Path(working_copy, package_list_file) + if package_list_file_path.exists(): + try: + package_list_str = parse_package_list_file(Path(working_copy, package_list_file)) + if package_list_file_name in ["pip3_packages","pip_packages"]: + package_list_str = convert_requirements_file(package_list_str) + except Exception as e: + print(f"Could not parse {Path(working_copy, package_list_file_path)}") + print(e) + break + return package_list_str + +# TODO add second flavor parameter to compare also different flavors +def compare_build_step(flavor_path:Path, build_step_path:Path, working_copy_1:Path, working_copy_1_name:str, working_copy_2:Path, working_copy_2_name:str): + package_list_file_name_alternatives = { + "python3_pip_packages": ["pip3_packages"], + "python2_pip_packages": ["pip_packages"] + } + result = {} packages_path = Path(build_step_path,"packages") if packages_path.is_dir(): for package_list_file in packages_path.iterdir(): - package_list_working_copy_str_1 = parse_package_list_file(Path(working_copy_1,package_list_file)) - package_list_working_copy_str_2 = parse_package_list_file(Path(working_copy_2,package_list_file)) - diff = compare_package_lists(package_list_working_copy_str_1, package_list_working_copy_str_2) - print(diff) + package_list_working_copy_str_1 = parse_package_list_file(Path(working_copy_1, package_list_file)) + package_list_working_copy_str_2 = load_package_file_or_alternative(working_copy_2, package_list_file, package_list_file_name_alternatives) + diff = compare_package_lists(package_list_working_copy_str_2, package_list_working_copy_str_1) + diff = diff.rename(columns={"Version1": f"Version in {working_copy_2_name}", "Version2": f"Version in {working_copy_1_name}"}) + result[package_list_file.name] = diff + return result - -def compare_flavor(flavor_path:Path, working_copy_1:Path, working_copy_2:Path): +# TODO add second flavor parameter to compare also different flavors +def compare_flavor(flavor_path:Path, working_copy_1:Path, working_copy_1_name:str, working_copy_2:Path, working_copy_2_name:str): flavor_base_path = Path(flavor_path,"flavor_base") + result = {} if flavor_base_path.is_dir(): for build_step_path in flavor_base_path.iterdir(): if build_step_path.is_dir(): - compare_build_step(flavor_path,build_step_path,working_copy_1,working_copy_2) + diffs = compare_build_step(flavor_path, build_step_path, working_copy_1, working_copy_1_name, working_copy_2, working_copy_2_name) + result[build_step_path.name]=diffs + return result -get_last_tag_command = ["git","describe","--abbrev=0","--tags"] -last_tag_result = subprocess.run(get_last_tag_command,stdout=subprocess.PIPE) -last_tag_result.check_returncode() -last_tag = last_tag_result.stdout.decode("UTF-8").strip() -print(last_tag) +def get_last_git_tag(): + get_last_tag_command = ["git","describe","--abbrev=0","--tags"] + last_tag_result = subprocess.run(get_last_tag_command,stdout=subprocess.PIPE) + last_tag_result.check_returncode() + last_tag = last_tag_result.stdout.decode("UTF-8").strip() + return last_tag -with TemporaryDirectory() as tmp_dir: - print(tmp_dir) +def checkout_git_tag_as_worktree(tmp_dir, last_tag): checkout_last_tag_command = ["git","worktree","add",tmp_dir,last_tag] checkout_last_tag_result = subprocess.run(checkout_last_tag_command,stderr=subprocess.PIPE) checkout_last_tag_result.check_returncode() init_submodule_command = ["git","submodule","update","--init"] init_submodule_result = subprocess.run(init_submodule_command,cwd=tmp_dir,stderr=subprocess.PIPE) init_submodule_result.check_returncode() + + +def get_package_list_diff_file_name(build_step, package_list): + return f"{build_step}_{package_list}_diff.md" + +def generate_dependency_diff_report_for_flavor(flavor_path_1:Path, flavor_path_2:Path, + working_copy_1_name:str, working_copy_2_name:str, + diffs, output_directory:Path): + flavor1 = flavor_path_1.name.capitalize() + flavor2 = flavor_path_2.name.capitalize() + overview_page = f"# Package Version Comparison between {flavor1} flavor in {working_copy_1_name} and {flavor2} flavor in {working_copy_2_name}\n\n" + for build_step in sorted(list(diffs.keys()),reverse=True): + if len(diffs[build_step]) > 0: + overview_page += f"- {build_step}\n" + for package_list in sorted(list(diffs[build_step].keys())): + package_list_name = " ".join(word.capitalize() for word in package_list.split("_")) + overview_page += f" - [{package_list_name}]({get_package_list_diff_file_name(build_step,package_list)})\n" + overview_file = Path(output_directory,"README.md") + with overview_file.open("wt") as f: + f.write(overview_page) + + +last_tag = get_last_git_tag() +output_directory = Path(tempfile.mkdtemp()) +print(output_directory, output_directory) +with TemporaryDirectory() as tmp_dir: + checkout_git_tag_as_worktree(tmp_dir, last_tag) working_copy_root = Path(".") + working_copy_1_name = "HEAD" + working_copy_2_name = last_tag for flavor_path in Path("flavors").iterdir(): if flavor_path.is_dir(): - compare_flavor(flavor_path.relative_to(working_copy_root),working_copy_root,tmp_dir) - + diffs = compare_flavor(flavor_path.relative_to(working_copy_root), working_copy_root, working_copy_1_name, tmp_dir, working_copy_2_name) + generate_dependency_diff_report_for_flavor(flavor_path, flavor_path, + working_copy_1_name, working_copy_2_name, + diffs, output_directory) -# package_list_1_path = sys.argv[1] -# package_list_2_path = sys.argv[2] -# package_list_1 = pd.read_csv(package_list_1_path, delimiter="|", names=["package","version"]) -# package_list_2 = pd.read_csv(package_list_2_path, delimiter="|", names=["package","version"]) -# diff = pd.merge(package_list_1, package_list_2, how='outer', on='package',sort=False) -# diff = diff.sort_values("package") -# diff["Updated"] = diff["version_x"]!=diff["version_y"] -# diff = diff.reset_index(drop=True) -# diff.to_markdown(sys.stdout) -# print() From c3f2cfdb7ebb395a7e5cc75545339ab7f155e286 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 21 Aug 2021 22:19:34 +0200 Subject: [PATCH 04/17] Add with-version parameter to install_via_pip.pl call in Dockerfiles --- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index d1720eeed..fac6602dd 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -13,4 +13,4 @@ RUN apt-get -y update && \ COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packeges --python-binary python3 +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packeges --python-binary python3 --with-versions --allow-no-version diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index d1720eeed..fac6602dd 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -13,4 +13,4 @@ RUN apt-get -y update && \ COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packeges --python-binary python3 +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packeges --python-binary python3 --with-versions --allow-no-version From 6bd69c146031498c56e802e8a9cef8f6a659388f Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 21 Aug 2021 22:25:29 +0200 Subject: [PATCH 05/17] Finish report generation and refactor diff collection to be used with two different flavors or build_steps --- scripts/generate_package_diffs_for_flavors.py | 349 ++++++++++++------ 1 file changed, 242 insertions(+), 107 deletions(-) diff --git a/scripts/generate_package_diffs_for_flavors.py b/scripts/generate_package_diffs_for_flavors.py index 1f29f6d12..da18afaf9 100644 --- a/scripts/generate_package_diffs_for_flavors.py +++ b/scripts/generate_package_diffs_for_flavors.py @@ -1,150 +1,285 @@ -import sys -import pandas as pd -import numpy as np +import re import subprocess +import tempfile +from io import StringIO from pathlib import Path from tempfile import TemporaryDirectory -from io import StringIO -import re -from typing import Dict, List -import tempfile +from typing import Dict, List, Optional, Tuple + +import pandas as pd -def parse_package_list_file(file_path:Path)->str: + +def parse_package_list_file(file_path: Path) -> str: parse_package_list_file_command = [ - "perl","ext/scripts/list_newest_versions/extract_columns_from_package_lisl.pl", - "--file",str(file_path),"--columns","0,1"] - result = subprocess.run(parse_package_list_file_command,stdout=subprocess.PIPE) + "perl", "ext/scripts/list_newest_versions/extract_columns_from_package_lisl.pl", + "--file", str(file_path), "--columns", "0,1"] + result = subprocess.run(parse_package_list_file_command, stdout=subprocess.PIPE) result.check_returncode() result_string = result.stdout.decode("UTF-8") return result_string -def compare_package_lists(package_list_1:str, package_list_2:str)->pd.DataFrame: - package_list_1_df = pd.read_csv(StringIO(package_list_1), delimiter="|", names=["Package","Version1"]) - package_list_1_df["Version1"]=package_list_1_df["Version1"].replace("<<<<1>>>>","No version specified") - package_list_2_df = pd.read_csv(StringIO(package_list_2), delimiter="|", names=["Package","Version2"]) - package_list_2_df["Version2"]=package_list_2_df["Version2"].replace("<<<<1>>>>","No version specified") - diff_df = pd.merge(package_list_1_df, package_list_2_df, how='outer', on='Package',sort=False) + +def compare_package_lists(package_list_1: str, package_list_2: str) -> pd.DataFrame: + package_list_1_df = pd.read_csv(StringIO(package_list_1), delimiter="|", names=["Package", "Version1"]) + package_list_1_df["Version1"] = package_list_1_df["Version1"].replace("<<<<1>>>>", "No version specified") + package_list_2_df = pd.read_csv(StringIO(package_list_2), delimiter="|", names=["Package", "Version2"]) + package_list_2_df["Version2"] = package_list_2_df["Version2"].replace("<<<<1>>>>", "No version specified") + diff_df = pd.merge(package_list_1_df, package_list_2_df, how='outer', on='Package', sort=False) new = diff_df["Version1"].isnull() & ~diff_df["Version2"].isnull() removed = diff_df["Version2"].isnull() & ~diff_df["Version1"].isnull() - updated = ~diff_df["Version1"].isnull() & ~diff_df["Version2"].isnull() & (diff_df["Version1"]!=diff_df["Version2"]) - diff_df["Status"]="" - diff_df["Status"].values[new]="NEW" - diff_df["Status"].values[removed]="REMOVED" - diff_df["Status"].values[updated]="UPDATED" + updated = ~diff_df["Version1"].isnull() & ~diff_df["Version2"].isnull() & ( + diff_df["Version1"] != diff_df["Version2"]) + diff_df["Status"] = "" + diff_df["Status"].values[new] = "NEW" + diff_df["Status"].values[removed] = "REMOVED" + diff_df["Status"].values[updated] = "UPDATED" diff_df = diff_df.fillna("") - diff_df = diff_df.sort_values("Status",ascending=False) + diff_df = diff_df.sort_values("Status", ascending=False) diff_df = diff_df.reset_index(drop=True) return diff_df -def convert_requirements_file(package_list_str): + +def convert_requirements_file(package_list_str: str) -> str: def convert_line(line): - line = line.replace("|<<<<1>>>>","|") - line = line.replace("==","|") - line = line.replace(">=","|") - line = line.replace("<=","|") - line = re.sub(r"\|$","",line) - if not "|" in line: + line = line.replace("|<<<<1>>>>", "|") + line = line.replace("==", "|") + line = line.replace(">=", "|") + line = line.replace("<=", "|") + line = re.sub(r"\|$", "", line) + if "|" not in line: line += "|<<<<1>>>>" return line + result = "\n".join(convert_line(line) for line in package_list_str.splitlines()) return result -def load_package_file_or_alternative(working_copy:Path, package_list_file:Path, package_list_file_name_alternatives:Dict[str,List[str]]): + +def find_package_file_or_alternative(working_copy: Path, + build_step_path: Path, + package_list_file_name: str, + package_list_file_name_alternatives: Dict[str, List[str]]) -> Optional[str]: + possible_package_list_file_names = [package_list_file_name] + if package_list_file_name in package_list_file_name_alternatives: + possible_package_list_file_names += package_list_file_name_alternatives[package_list_file_name] + packages_directory = Path(working_copy, build_step_path, "packages") + if packages_directory.exists(): + for package_list_file_name in possible_package_list_file_names: + package_list_file_path = Path(packages_directory, package_list_file_name) + if package_list_file_path.exists(): + return package_list_file_name + return None + + +def load_package_file_or_alternative(working_copy: Path, + package_list_file: Path): package_list_str = "" - possible_package_list_file_names = [package_list_file.name] - if package_list_file.name in package_list_file_name_alternatives: - possible_package_list_file_names += package_list_file_name_alternatives[package_list_file.name] - for package_list_file_name in possible_package_list_file_names: - package_list_file = Path(package_list_file.parent, package_list_file_name) - package_list_file_path = Path(working_copy, package_list_file) - if package_list_file_path.exists(): - try: - package_list_str = parse_package_list_file(Path(working_copy, package_list_file)) - if package_list_file_name in ["pip3_packages","pip_packages"]: - package_list_str = convert_requirements_file(package_list_str) - except Exception as e: - print(f"Could not parse {Path(working_copy, package_list_file_path)}") - print(e) - break + try: + package_list_str = parse_package_list_file(Path(working_copy, package_list_file)) + if package_list_file.name in ["pip3_packages", "pip_packages"]: + package_list_str = convert_requirements_file(package_list_str) + except Exception as e: + print(f"Could not parse {Path(working_copy, package_list_file)}") + print(e) return package_list_str -# TODO add second flavor parameter to compare also different flavors -def compare_build_step(flavor_path:Path, build_step_path:Path, working_copy_1:Path, working_copy_1_name:str, working_copy_2:Path, working_copy_2_name:str): + +def compare_build_step(build_step_path_1: Path, working_copy_1: Path, working_copy_1_name: str, + build_step_path_2: Path, working_copy_2: Path, working_copy_2_name: str) \ + -> Dict[Tuple[str, Optional[str]], pd.DataFrame]: package_list_file_name_alternatives = { - "python3_pip_packages": ["pip3_packages"], - "python2_pip_packages": ["pip_packages"] - } + "python3_pip_packages": ["pip3_packages"], + "python2_pip_packages": ["pip_packages"] + } result = {} - packages_path = Path(build_step_path,"packages") - if packages_path.is_dir(): - for package_list_file in packages_path.iterdir(): - package_list_working_copy_str_1 = parse_package_list_file(Path(working_copy_1, package_list_file)) - package_list_working_copy_str_2 = load_package_file_or_alternative(working_copy_2, package_list_file, package_list_file_name_alternatives) - diff = compare_package_lists(package_list_working_copy_str_2, package_list_working_copy_str_1) - diff = diff.rename(columns={"Version1": f"Version in {working_copy_2_name}", "Version2": f"Version in {working_copy_1_name}"}) - result[package_list_file.name] = diff + packages_path_1 = Path(build_step_path_1, "packages") + if packages_path_1.is_dir(): + for package_list_file_1 in packages_path_1.iterdir(): + package_list_file_name_1 = package_list_file_1.name + package_list_working_copy_str_1 = parse_package_list_file(Path(working_copy_1, package_list_file_1)) + package_list_file_name_2 = find_package_file_or_alternative(working_copy_2, + build_step_path_2, + package_list_file_name_1, + package_list_file_name_alternatives) + result_key = (package_list_file_name_1, package_list_file_name_2) + if package_list_file_name_2 is None: + package_list_working_copy_str_2 = "" + else: + package_list_file_2 = Path(build_step_path_2, "packages", package_list_file_name_2) + package_list_working_copy_str_2 = load_package_file_or_alternative(working_copy_2, + package_list_file_2) + diff_df = compare_package_lists(package_list_working_copy_str_2, package_list_working_copy_str_1) + diff_df = diff_df.rename(columns={"Version1": f"Version in {working_copy_2_name}", + "Version2": f"Version in {working_copy_1_name}"}) + result[result_key] = diff_df return result -# TODO add second flavor parameter to compare also different flavors -def compare_flavor(flavor_path:Path, working_copy_1:Path, working_copy_1_name:str, working_copy_2:Path, working_copy_2_name:str): - flavor_base_path = Path(flavor_path,"flavor_base") + +def compare_flavor(flavor_path_1: Path, working_copy_1: Path, working_copy_1_name: str, + flavor_path_2: Path, working_copy_2: Path, working_copy_2_name: str) \ + -> Dict[Tuple[str, str], Dict[Tuple[str, Optional[str]], pd.DataFrame]]: + flavor_base_path_1 = Path(flavor_path_1, "flavor_base") + flavor_base_path_2 = Path(flavor_path_2, "flavor_base") result = {} - if flavor_base_path.is_dir(): - for build_step_path in flavor_base_path.iterdir(): - if build_step_path.is_dir(): - diffs = compare_build_step(flavor_path, build_step_path, working_copy_1, working_copy_1_name, working_copy_2, working_copy_2_name) - result[build_step_path.name]=diffs + if flavor_base_path_1.is_dir(): + for build_step_path_1 in flavor_base_path_1.iterdir(): + if build_step_path_1.is_dir(): + build_step_name_1 = build_step_path_1.name + build_step_name_2 = build_step_name_1 + build_step_path_2 = Path(flavor_base_path_2, build_step_name_2) + diffs = compare_build_step(build_step_path_1, working_copy_1, working_copy_1_name, + build_step_path_2, working_copy_2, working_copy_2_name) + result[(build_step_name_1, build_step_name_2)] = diffs return result -def get_last_git_tag(): - get_last_tag_command = ["git","describe","--abbrev=0","--tags"] - last_tag_result = subprocess.run(get_last_tag_command,stdout=subprocess.PIPE) +def get_last_git_tag() -> str: + get_last_tag_command = ["git", "describe", "--abbrev=0", "--tags"] + last_tag_result = subprocess.run(get_last_tag_command, stdout=subprocess.PIPE) last_tag_result.check_returncode() last_tag = last_tag_result.stdout.decode("UTF-8").strip() return last_tag + def checkout_git_tag_as_worktree(tmp_dir, last_tag): - checkout_last_tag_command = ["git","worktree","add",tmp_dir,last_tag] - checkout_last_tag_result = subprocess.run(checkout_last_tag_command,stderr=subprocess.PIPE) + checkout_last_tag_command = ["git", "worktree", "add", tmp_dir, last_tag] + checkout_last_tag_result = subprocess.run(checkout_last_tag_command, stderr=subprocess.PIPE) checkout_last_tag_result.check_returncode() - init_submodule_command = ["git","submodule","update","--init"] - init_submodule_result = subprocess.run(init_submodule_command,cwd=tmp_dir,stderr=subprocess.PIPE) + init_submodule_command = ["git", "submodule", "update", "--init"] + init_submodule_result = subprocess.run(init_submodule_command, cwd=tmp_dir, stderr=subprocess.PIPE) init_submodule_result.check_returncode() -def get_package_list_diff_file_name(build_step, package_list): - return f"{build_step}_{package_list}_diff.md" - -def generate_dependency_diff_report_for_flavor(flavor_path_1:Path, flavor_path_2:Path, - working_copy_1_name:str, working_copy_2_name:str, - diffs, output_directory:Path): - flavor1 = flavor_path_1.name.capitalize() - flavor2 = flavor_path_2.name.capitalize() - overview_page = f"# Package Version Comparison between {flavor1} flavor in {working_copy_1_name} and {flavor2} flavor in {working_copy_2_name}\n\n" - for build_step in sorted(list(diffs.keys()),reverse=True): - if len(diffs[build_step]) > 0: - overview_page += f"- {build_step}\n" - for package_list in sorted(list(diffs[build_step].keys())): - package_list_name = " ".join(word.capitalize() for word in package_list.split("_")) - overview_page += f" - [{package_list_name}]({get_package_list_diff_file_name(build_step,package_list)})\n" - overview_file = Path(output_directory,"README.md") - with overview_file.open("wt") as f: - f.write(overview_page) - - -last_tag = get_last_git_tag() -output_directory = Path(tempfile.mkdtemp()) -print(output_directory, output_directory) -with TemporaryDirectory() as tmp_dir: - checkout_git_tag_as_worktree(tmp_dir, last_tag) - working_copy_root = Path(".") - working_copy_1_name = "HEAD" - working_copy_2_name = last_tag - for flavor_path in Path("flavors").iterdir(): +def generate_dependency_diff_report_for_package_list( + package_file_diff_file: Path, + diff_df: pd.DataFrame): + package_file_diff_file.parent.mkdir(parents=True, exist_ok=True) + with package_file_diff_file.open("wt") as f: + diff_df.to_markdown(f) + + +def generate_dependency_diff_report_for_build_step( + build_steps: Tuple[str, str], + diffs: Dict[Tuple[str, Optional[str]], pd.DataFrame], + base_output_directory: Path, + relative_output_directory: Path): + result = "" + if len(diffs) > 0: + build_step_caption = generate_build_step_caption(build_steps) + result = f"- {build_step_caption}\n" + for package_lists in sorted(list(diffs.keys())): + package_list_caption = \ + generate_package_list_caption(package_lists) + relative_package_file_diff_file = Path(relative_output_directory, f"{package_lists[0]}_diff.md") + result += f" - [{package_list_caption}]({relative_package_file_diff_file})\n" + package_file_diff_file = Path(base_output_directory, relative_package_file_diff_file) + generate_dependency_diff_report_for_package_list( + package_file_diff_file, diffs[package_lists]) + return result + + +def generate_build_step_caption(build_steps): + build_step_1_capitalized = build_steps[0].capitalize() + if build_steps[1] is None or build_steps[0] == build_steps[1]: + build_step_caption = f"Comparison of build step {build_step_1_capitalized}" + else: + build_step_2_capitalized = build_steps[1].capitalize() + build_step_caption = f"Comparison of build steps {build_step_1_capitalized} and {build_step_2_capitalized}" + return build_step_caption + + +def generate_package_list_caption(package_lists: Tuple[str, Optional[str]], ) -> str: + package_list_name_1 = " ".join(word.capitalize() for word in package_lists[0].split("_")) + if package_lists[1] is None or package_lists[0] == package_lists[1]: + if package_lists[0] == package_lists[1]: + package_list_caption = f"Comparison of package list {package_list_name_1}" + else: + package_list_caption = f"New package list {package_list_name_1}" + else: + package_list_name_2 = " ".join(word.capitalize() for word in package_lists[1].split("_")) + package_list_caption = f"Comparison of package lists {package_list_name_1} and {package_list_name_2}" + return package_list_caption + + +def generate_dependency_diff_report_for_flavor(flavor_name_1: str, working_copy_1_name: str, + flavor_name_2: str, working_copy_2_name: str, + diffs: Dict[ + Tuple[str, str], Dict[Tuple[str, Optional[str]], pd.DataFrame]], + base_output_directory: Path, relative_output_directory: Path): + relative_overview_file = Path(relative_output_directory, "README.md") + overview_file = Path(base_output_directory, relative_overview_file) + overview_file.parent.mkdir(parents=True, exist_ok=True) + flavor_name_1_capitalized = flavor_name_1.capitalize() + flavor_name_2_capitalized = flavor_name_2.capitalize() + overview_file_content = \ + f"# Package Version Comparison between " \ + f"{flavor_name_1_capitalized} flavor in {working_copy_1_name} and " \ + f"{flavor_name_2_capitalized} flavor in {working_copy_2_name}\n\n" + if flavor_name_1 == flavor_name_1: + result = f"- [Comparison of flavor {flavor_name_1_capitalized}" \ + f"]({relative_overview_file})\n" + else: + result = f"- [Comparison of flavors " \ + f"{flavor_name_1_capitalized} and {flavor_name_2_capitalized}" \ + f"]({relative_overview_file})\n" + for build_steps in sorted(list(diffs.keys()), reverse=True): + build_step_base_output_directory = Path(base_output_directory, relative_output_directory) + build_step_relative_output_directory = Path(build_steps[0]) + overview_file_content += \ + generate_dependency_diff_report_for_build_step( + build_steps, + diffs[build_steps], + build_step_base_output_directory, + build_step_relative_output_directory) + with overview_file.open("wt") as f: + f.write(overview_file_content) + return result + + +def generate_dependency_diff_report_for_all_flavors(working_copy_1_root: Path, + working_copy_1_name: str, + working_copy_2_root: Path, + working_copy_2_name: str, + base_output_directory: Path): + base_output_directory.mkdir(parents=True, exist_ok=True) + overview_file = Path(base_output_directory, "README.md") + overview_file_content = \ + f"# Package Version Comparison between " \ + f"{working_copy_1_name} and " \ + f"{working_copy_2_name}\n\n" + for flavor_path in Path(working_copy_1_root, "flavors").iterdir(): if flavor_path.is_dir(): - diffs = compare_flavor(flavor_path.relative_to(working_copy_root), working_copy_root, working_copy_1_name, tmp_dir, working_copy_2_name) - generate_dependency_diff_report_for_flavor(flavor_path, flavor_path, - working_copy_1_name, working_copy_2_name, - diffs, output_directory) + relative_flavor_path = flavor_path.relative_to(working_copy_1_root) + diffs = compare_flavor(relative_flavor_path, working_copy_1_root, working_copy_1_name, + relative_flavor_path, working_copy_2_root, working_copy_2_name) + if len(diffs) > 0: + flavor_1 = flavor_path.name + flavor_2 = flavor_path.name + if flavor_1 == flavor_2: + flavor_relative_output_directory = Path(flavor_1) + else: + flavor_relative_output_directory = Path(f"{flavor_1}__{flavor_2}") + overview_file_content += \ + generate_dependency_diff_report_for_flavor(flavor_1, working_copy_1_name, + flavor_2, working_copy_2_name, + diffs, + base_output_directory, + flavor_relative_output_directory) + with overview_file.open("wt") as f: + f.write(overview_file_content) + +def main(): + last_tag = get_last_git_tag() + output_directory = Path(tempfile.mkdtemp()) + print("output_directory", output_directory) + with TemporaryDirectory() as working_copy_2_root: + checkout_git_tag_as_worktree(working_copy_2_root, last_tag) + working_copy_root = Path(".") + working_copy_1_name = "HEAD" + working_copy_2_name = last_tag + generate_dependency_diff_report_for_all_flavors(working_copy_root, working_copy_1_name, + working_copy_2_root, working_copy_2_name, + output_directory) + +if __name__ == '__main__': + main() From 0257ef8e30f5642472a0c51d736dc7e2119a39d4 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 21 Aug 2021 22:51:11 +0200 Subject: [PATCH 06/17] Moved generate_package_diffs_for_flavors.py --- .../generate_package_diffs_for_flavors.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{ => generate_package_diffs_for_flavors}/generate_package_diffs_for_flavors.py (100%) diff --git a/scripts/generate_package_diffs_for_flavors.py b/scripts/generate_package_diffs_for_flavors/generate_package_diffs_for_flavors.py similarity index 100% rename from scripts/generate_package_diffs_for_flavors.py rename to scripts/generate_package_diffs_for_flavors/generate_package_diffs_for_flavors.py From e64df9ca2ddb9d88970cf23ae151b2229a79baf6 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 21 Aug 2021 22:51:49 +0200 Subject: [PATCH 07/17] Add poetry setup to generate_package_diffs_for_flavors.py and cli via click --- .../generate_package_diffs_for_flavors.py | 15 +- .../poetry.lock | 224 ++++++++++++++++++ .../pyproject.toml | 17 ++ 3 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 scripts/generate_package_diffs_for_flavors/poetry.lock create mode 100644 scripts/generate_package_diffs_for_flavors/pyproject.toml diff --git a/scripts/generate_package_diffs_for_flavors/generate_package_diffs_for_flavors.py b/scripts/generate_package_diffs_for_flavors/generate_package_diffs_for_flavors.py index da18afaf9..66b1f4f38 100644 --- a/scripts/generate_package_diffs_for_flavors/generate_package_diffs_for_flavors.py +++ b/scripts/generate_package_diffs_for_flavors/generate_package_diffs_for_flavors.py @@ -6,6 +6,7 @@ from tempfile import TemporaryDirectory from typing import Dict, List, Optional, Tuple +import click import pandas as pd @@ -267,18 +268,22 @@ def generate_dependency_diff_report_for_all_flavors(working_copy_1_root: Path, with overview_file.open("wt") as f: f.write(overview_file_content) -def main(): +@click.command() +@click.option('--output-directory', required=True, help="Directory where the diff reports are generated", + type=click.Path(exists=False)) +@click.option('--current-working-copy-name', required=True, help="Name of the current git working copy. " + "For example, the version of a new relaase.", + type=str) +def main(output_directory:str, current_working_copy_name:str): last_tag = get_last_git_tag() - output_directory = Path(tempfile.mkdtemp()) - print("output_directory", output_directory) with TemporaryDirectory() as working_copy_2_root: checkout_git_tag_as_worktree(working_copy_2_root, last_tag) working_copy_root = Path(".") - working_copy_1_name = "HEAD" + working_copy_1_name = current_working_copy_name working_copy_2_name = last_tag generate_dependency_diff_report_for_all_flavors(working_copy_root, working_copy_1_name, working_copy_2_root, working_copy_2_name, - output_directory) + Path(output_directory)) if __name__ == '__main__': diff --git a/scripts/generate_package_diffs_for_flavors/poetry.lock b/scripts/generate_package_diffs_for_flavors/poetry.lock new file mode 100644 index 000000000..72335cb9f --- /dev/null +++ b/scripts/generate_package_diffs_for_flavors/poetry.lock @@ -0,0 +1,224 @@ +[[package]] +name = "click" +version = "8.0.1" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "importlib-metadata" +version = "4.6.4" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +perf = ["ipython"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] + +[[package]] +name = "numpy" +version = "1.19.5" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "pandas" +version = "1.1.5" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +numpy = ">=1.15.4" +python-dateutil = ">=2.7.3" +pytz = ">=2017.2" + +[package.extras] +test = ["pytest (>=4.0.2)", "pytest-xdist", "hypothesis (>=3.58)"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2021.1" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tabulate" +version = "0.8.9" +description = "Pretty-print tabular data" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +widechars = ["wcwidth"] + +[[package]] +name = "typing-extensions" +version = "3.10.0.0" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "zipp" +version = "3.5.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[metadata] +lock-version = "1.1" +python-versions = ">=3.6.1" +content-hash = "054c7bed17cd1377e1c8656304777ddbf9b4bf8c99a3108a0e53ec5c00e7f25b" + +[metadata.files] +click = [ + {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, + {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +importlib-metadata = [ + {file = "importlib_metadata-4.6.4-py3-none-any.whl", hash = "sha256:ed5157fef23a4bc4594615a0dd8eba94b2bb36bf2a343fa3d8bb2fa0a62a99d5"}, + {file = "importlib_metadata-4.6.4.tar.gz", hash = "sha256:7b30a78db2922d78a6f47fb30683156a14f3c6aa5cc23f77cc8967e9ab2d002f"}, +] +numpy = [ + {file = "numpy-1.19.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc6bd4fd593cb261332568485e20a0712883cf631f6f5e8e86a52caa8b2b50ff"}, + {file = "numpy-1.19.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:aeb9ed923be74e659984e321f609b9ba54a48354bfd168d21a2b072ed1e833ea"}, + {file = "numpy-1.19.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8b5e972b43c8fc27d56550b4120fe6257fdc15f9301914380b27f74856299fea"}, + {file = "numpy-1.19.5-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:43d4c81d5ffdff6bae58d66a3cd7f54a7acd9a0e7b18d97abb255defc09e3140"}, + {file = "numpy-1.19.5-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:a4646724fba402aa7504cd48b4b50e783296b5e10a524c7a6da62e4a8ac9698d"}, + {file = "numpy-1.19.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2e55195bc1c6b705bfd8ad6f288b38b11b1af32f3c8289d6c50d47f950c12e76"}, + {file = "numpy-1.19.5-cp36-cp36m-win32.whl", hash = "sha256:39b70c19ec771805081578cc936bbe95336798b7edf4732ed102e7a43ec5c07a"}, + {file = "numpy-1.19.5-cp36-cp36m-win_amd64.whl", hash = "sha256:dbd18bcf4889b720ba13a27ec2f2aac1981bd41203b3a3b27ba7a33f88ae4827"}, + {file = "numpy-1.19.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:603aa0706be710eea8884af807b1b3bc9fb2e49b9f4da439e76000f3b3c6ff0f"}, + {file = "numpy-1.19.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:cae865b1cae1ec2663d8ea56ef6ff185bad091a5e33ebbadd98de2cfa3fa668f"}, + {file = "numpy-1.19.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:36674959eed6957e61f11c912f71e78857a8d0604171dfd9ce9ad5cbf41c511c"}, + {file = "numpy-1.19.5-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:06fab248a088e439402141ea04f0fffb203723148f6ee791e9c75b3e9e82f080"}, + {file = "numpy-1.19.5-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6149a185cece5ee78d1d196938b2a8f9d09f5a5ebfbba66969302a778d5ddd1d"}, + {file = "numpy-1.19.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:50a4a0ad0111cc1b71fa32dedd05fa239f7fb5a43a40663269bb5dc7877cfd28"}, + {file = "numpy-1.19.5-cp37-cp37m-win32.whl", hash = "sha256:d051ec1c64b85ecc69531e1137bb9751c6830772ee5c1c426dbcfe98ef5788d7"}, + {file = "numpy-1.19.5-cp37-cp37m-win_amd64.whl", hash = "sha256:a12ff4c8ddfee61f90a1633a4c4afd3f7bcb32b11c52026c92a12e1325922d0d"}, + {file = "numpy-1.19.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cf2402002d3d9f91c8b01e66fbb436a4ed01c6498fffed0e4c7566da1d40ee1e"}, + {file = "numpy-1.19.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1ded4fce9cfaaf24e7a0ab51b7a87be9038ea1ace7f34b841fe3b6894c721d1c"}, + {file = "numpy-1.19.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94"}, + {file = "numpy-1.19.5-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:759e4095edc3c1b3ac031f34d9459fa781777a93ccc633a472a5468587a190ff"}, + {file = "numpy-1.19.5-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:a9d17f2be3b427fbb2bce61e596cf555d6f8a56c222bd2ca148baeeb5e5c783c"}, + {file = "numpy-1.19.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:99abf4f353c3d1a0c7a5f27699482c987cf663b1eac20db59b8c7b061eabd7fc"}, + {file = "numpy-1.19.5-cp38-cp38-win32.whl", hash = "sha256:384ec0463d1c2671170901994aeb6dce126de0a95ccc3976c43b0038a37329c2"}, + {file = "numpy-1.19.5-cp38-cp38-win_amd64.whl", hash = "sha256:811daee36a58dc79cf3d8bdd4a490e4277d0e4b7d103a001a4e73ddb48e7e6aa"}, + {file = "numpy-1.19.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c843b3f50d1ab7361ca4f0b3639bf691569493a56808a0b0c54a051d260b7dbd"}, + {file = "numpy-1.19.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d6631f2e867676b13026e2846180e2c13c1e11289d67da08d71cacb2cd93d4aa"}, + {file = "numpy-1.19.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7fb43004bce0ca31d8f13a6eb5e943fa73371381e53f7074ed21a4cb786c32f8"}, + {file = "numpy-1.19.5-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:2ea52bd92ab9f768cc64a4c3ef8f4b2580a17af0a5436f6126b08efbd1838371"}, + {file = "numpy-1.19.5-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:400580cbd3cff6ffa6293df2278c75aef2d58d8d93d3c5614cd67981dae68ceb"}, + {file = "numpy-1.19.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:df609c82f18c5b9f6cb97271f03315ff0dbe481a2a02e56aeb1b1a985ce38e60"}, + {file = "numpy-1.19.5-cp39-cp39-win32.whl", hash = "sha256:ab83f24d5c52d60dbc8cd0528759532736b56db58adaa7b5f1f76ad551416a1e"}, + {file = "numpy-1.19.5-cp39-cp39-win_amd64.whl", hash = "sha256:0eef32ca3132a48e43f6a0f5a82cb508f22ce5a3d6f67a8329c81c8e226d3f6e"}, + {file = "numpy-1.19.5-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:a0d53e51a6cb6f0d9082decb7a4cb6dfb33055308c4c44f53103c073f649af73"}, + {file = "numpy-1.19.5.zip", hash = "sha256:a76f502430dd98d7546e1ea2250a7360c065a5fdea52b2dffe8ae7180909b6f4"}, +] +pandas = [ + {file = "pandas-1.1.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:bf23a3b54d128b50f4f9d4675b3c1857a688cc6731a32f931837d72effb2698d"}, + {file = "pandas-1.1.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5a780260afc88268a9d3ac3511d8f494fdcf637eece62fb9eb656a63d53eb7ca"}, + {file = "pandas-1.1.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:b61080750d19a0122469ab59b087380721d6b72a4e7d962e4d7e63e0c4504814"}, + {file = "pandas-1.1.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:0de3ddb414d30798cbf56e642d82cac30a80223ad6fe484d66c0ce01a84d6f2f"}, + {file = "pandas-1.1.5-cp36-cp36m-win32.whl", hash = "sha256:70865f96bb38fec46f7ebd66d4b5cfd0aa6b842073f298d621385ae3898d28b5"}, + {file = "pandas-1.1.5-cp36-cp36m-win_amd64.whl", hash = "sha256:19a2148a1d02791352e9fa637899a78e371a3516ac6da5c4edc718f60cbae648"}, + {file = "pandas-1.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:26fa92d3ac743a149a31b21d6f4337b0594b6302ea5575b37af9ca9611e8981a"}, + {file = "pandas-1.1.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c16d59c15d946111d2716856dd5479221c9e4f2f5c7bc2d617f39d870031e086"}, + {file = "pandas-1.1.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:3be7a7a0ca71a2640e81d9276f526bca63505850add10206d0da2e8a0a325dae"}, + {file = "pandas-1.1.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:573fba5b05bf2c69271a32e52399c8de599e4a15ab7cec47d3b9c904125ab788"}, + {file = "pandas-1.1.5-cp37-cp37m-win32.whl", hash = "sha256:21b5a2b033380adbdd36b3116faaf9a4663e375325831dac1b519a44f9e439bb"}, + {file = "pandas-1.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:24c7f8d4aee71bfa6401faeba367dd654f696a77151a8a28bc2013f7ced4af98"}, + {file = "pandas-1.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2860a97cbb25444ffc0088b457da0a79dc79f9c601238a3e0644312fcc14bf11"}, + {file = "pandas-1.1.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5008374ebb990dad9ed48b0f5d0038124c73748f5384cc8c46904dace27082d9"}, + {file = "pandas-1.1.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2c2f7c670ea4e60318e4b7e474d56447cf0c7d83b3c2a5405a0dbb2600b9c48e"}, + {file = "pandas-1.1.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0a643bae4283a37732ddfcecab3f62dd082996021b980f580903f4e8e01b3c5b"}, + {file = "pandas-1.1.5-cp38-cp38-win32.whl", hash = "sha256:5447ea7af4005b0daf695a316a423b96374c9c73ffbd4533209c5ddc369e644b"}, + {file = "pandas-1.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:4c62e94d5d49db116bef1bd5c2486723a292d79409fc9abd51adf9e05329101d"}, + {file = "pandas-1.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:731568be71fba1e13cae212c362f3d2ca8932e83cb1b85e3f1b4dd77d019254a"}, + {file = "pandas-1.1.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:c61c043aafb69329d0f961b19faa30b1dab709dd34c9388143fc55680059e55a"}, + {file = "pandas-1.1.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:2b1c6cd28a0dfda75c7b5957363333f01d370936e4c6276b7b8e696dd500582a"}, + {file = "pandas-1.1.5-cp39-cp39-win32.whl", hash = "sha256:c94ff2780a1fd89f190390130d6d36173ca59fcfb3fe0ff596f9a56518191ccb"}, + {file = "pandas-1.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:edda9bacc3843dfbeebaf7a701763e68e741b08fccb889c003b0a52f0ee95782"}, + {file = "pandas-1.1.5.tar.gz", hash = "sha256:f10fc41ee3c75a474d3bdf68d396f10782d013d7f67db99c0efbfd0acb99701b"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +pytz = [ + {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"}, + {file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +tabulate = [ + {file = "tabulate-0.8.9-py3-none-any.whl", hash = "sha256:d7c013fe7abbc5e491394e10fa845f8f32fe54f8dc60c6622c6cf482d25d47e4"}, + {file = "tabulate-0.8.9.tar.gz", hash = "sha256:eb1d13f25760052e8931f2ef80aaf6045a6cceb47514db8beab24cded16f13a7"}, +] +typing-extensions = [ + {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, + {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"}, + {file = "typing_extensions-3.10.0.0.tar.gz", hash = "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342"}, +] +zipp = [ + {file = "zipp-3.5.0-py3-none-any.whl", hash = "sha256:957cfda87797e389580cb8b9e3870841ca991e2125350677b2ca83a0e99390a3"}, + {file = "zipp-3.5.0.tar.gz", hash = "sha256:f5812b1e007e48cff63449a5e9f4e7ebea716b4111f9c4f9a645f91d579bf0c4"}, +] diff --git a/scripts/generate_package_diffs_for_flavors/pyproject.toml b/scripts/generate_package_diffs_for_flavors/pyproject.toml new file mode 100644 index 000000000..69687d2a5 --- /dev/null +++ b/scripts/generate_package_diffs_for_flavors/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "generate_package_diffs_for_flavors" +version = "0.1.0" +description = "" +authors = ["Torsten Kilias "] + +[tool.poetry.dependencies] +python = ">=3.6.1" +pandas = "^1.1.0" +tabulate = "^0.8.9" +click = "^8.0.1" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" From a418dfaf96c0fa9ead05f4da1f8732189ca7547d Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 11:48:00 +0200 Subject: [PATCH 08/17] Fix Dockerfiles --- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index fac6602dd..a1f30870d 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -13,4 +13,4 @@ RUN apt-get -y update && \ COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packeges --python-binary python3 --with-versions --allow-no-version +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index fac6602dd..a1f30870d 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -13,4 +13,4 @@ RUN apt-get -y update && \ COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packeges --python-binary python3 --with-versions --allow-no-version +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version From 3226aa05e0b3167f063e8ff6ed388f3b7b000258 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 11:59:30 +0200 Subject: [PATCH 09/17] Fix Dockerfiles --- .../flavor_base/language_deps/Dockerfile | 2 +- .../flavor_base/language_deps/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index e238defab..6ce8ca5fc 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -20,7 +20,7 @@ RUN /scripts/install_scripts/install_python3.6_pip.sh COPY scripts /scripts COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packges --python-binary python3 +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 ENV PYTHON3_PREFIX /usr ENV PYTHON3_VERSION python3.6 diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index e238defab..6ce8ca5fc 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -20,7 +20,7 @@ RUN /scripts/install_scripts/install_python3.6_pip.sh COPY scripts /scripts COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packges --python-binary python3 +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 ENV PYTHON3_PREFIX /usr ENV PYTHON3_VERSION python3.6 From 67a2006c3710c2d1a650e3ae06e1080809766564 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 12:54:02 +0200 Subject: [PATCH 10/17] Fix Dockerfiles --- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index a1f30870d..bc5441aed 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -1,6 +1,6 @@ FROM {{ language_deps }} -RUN mkdir -p /build_info/packages +RUN mkdir -p /build_info/packages/flavor_base_deps COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps RUN apt-get -y update && \ diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index a1f30870d..bc5441aed 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -1,6 +1,6 @@ FROM {{ language_deps }} -RUN mkdir -p /build_info/packages +RUN mkdir -p /build_info/packages/flavor_base_deps COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps RUN apt-get -y update && \ From 1c78201b23dfc8c34c65f2aede2434e6ba68d82d Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 13:01:22 +0200 Subject: [PATCH 11/17] Remove duplicated copy --- .../flavor_base/language_deps/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index 6ce8ca5fc..01bab35ed 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -18,7 +18,6 @@ RUN apt-get update && \ COPY scripts /scripts RUN /scripts/install_scripts/install_python3.6_pip.sh -COPY scripts /scripts COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 From d0c44e2575deae83b91c82e00cf522ef221aebd1 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 13:18:51 +0200 Subject: [PATCH 12/17] Replace manual apt-get installation in Dockerfiles (this was some form of regression) --- .../flavor_base/language_deps/Dockerfile | 13 +------------ .../flavor_base/language_deps/Dockerfile | 13 +------------ 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index 01bab35ed..388b512e6 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -3,19 +3,8 @@ FROM {{udfclient_deps}} RUN mkdir -p /build_info/packages/language_deps COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps -# ppa:deadsnakes/ppa needed for python3.6 under ubuntu:16.04 -RUN apt-get update && \ - apt-get -y install software-properties-common && \ - add-apt-repository ppa:deadsnakes/ppa && \ - apt-get -y update && \ - apt-get -y install $(cat /build_info/packages/language_deps/apt_get_packages) && \ - locale-gen en_US.UTF-8 && \ - update-locale LC_ALL=en_US.UTF-8 && \ - apt-get -y clean && \ - apt-get -y autoremove && \ - ldconfig +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages -COPY scripts /scripts RUN /scripts/install_scripts/install_python3.6_pip.sh COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index 6ce8ca5fc..8b34626fe 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -3,19 +3,8 @@ FROM {{udfclient_deps}} RUN mkdir -p /build_info/packages/language_deps COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps -# ppa:deadsnakes/ppa needed for python3.6 under ubuntu:16.04 -RUN apt-get update && \ - apt-get -y install software-properties-common && \ - add-apt-repository ppa:deadsnakes/ppa && \ - apt-get -y update && \ - apt-get -y install $(cat /build_info/packages/language_deps/apt_get_packages) && \ - locale-gen en_US.UTF-8 && \ - update-locale LC_ALL=en_US.UTF-8 && \ - apt-get -y clean && \ - apt-get -y autoremove && \ - ldconfig +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages -COPY scripts /scripts RUN /scripts/install_scripts/install_python3.6_pip.sh COPY scripts /scripts From 39ca16b30bbf487a8670abff874e047e81a5c3e3 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 13:29:10 +0200 Subject: [PATCH 13/17] Also replace manual apt-get installation in other Dockerfiles and add --with-version and --allow-no-version to apt install script calls --- .../flavor_base/flavor_base_deps/Dockerfile | 11 ++--------- .../flavor_base/language_deps/Dockerfile | 6 +++--- .../flavor_base/flavor_base_deps/Dockerfile | 11 ++--------- .../flavor_base/language_deps/Dockerfile | 7 +++---- 4 files changed, 10 insertions(+), 25 deletions(-) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index bc5441aed..a0e27f2d9 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -1,16 +1,9 @@ FROM {{ language_deps }} RUN mkdir -p /build_info/packages/flavor_base_deps -COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps - -RUN apt-get -y update && \ - apt-get -y install $(cat /build_info/packages/flavor_base_deps/apt_get_packages) && \ - locale-gen en_US.UTF-8 && \ - update-locale LC_ALL=en_US.UTF-8 && \ - apt-get -y clean && \ - apt-get -y autoremove && \ - ldconfig +COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions --allow-no-version COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index 388b512e6..47ad40b56 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -1,14 +1,14 @@ FROM {{udfclient_deps}} RUN mkdir -p /build_info/packages/language_deps -COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages +COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions --allow-no-version RUN /scripts/install_scripts/install_python3.6_pip.sh COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version ENV PYTHON3_PREFIX /usr ENV PYTHON3_VERSION python3.6 diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index bc5441aed..a0e27f2d9 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -1,16 +1,9 @@ FROM {{ language_deps }} RUN mkdir -p /build_info/packages/flavor_base_deps -COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps - -RUN apt-get -y update && \ - apt-get -y install $(cat /build_info/packages/flavor_base_deps/apt_get_packages) && \ - locale-gen en_US.UTF-8 && \ - update-locale LC_ALL=en_US.UTF-8 && \ - apt-get -y clean && \ - apt-get -y autoremove && \ - ldconfig +COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions --allow-no-version COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile index 8b34626fe..47ad40b56 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/language_deps/Dockerfile @@ -1,15 +1,14 @@ FROM {{udfclient_deps}} RUN mkdir -p /build_info/packages/language_deps -COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages +COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions --allow-no-version RUN /scripts/install_scripts/install_python3.6_pip.sh -COPY scripts /scripts COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version ENV PYTHON3_PREFIX /usr ENV PYTHON3_VERSION python3.6 From 651cf3046144f7c9a6ba6287edba4ddf2d124c3c Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 13:37:45 +0200 Subject: [PATCH 14/17] Replace manual apt-get installation in udfclient_deps of the cuda flavor --- .../flavor_base/udfclient_deps/Dockerfile | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/udfclient_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/udfclient_deps/Dockerfile index ea6d54bc5..f08c67ca7 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/udfclient_deps/Dockerfile @@ -6,18 +6,12 @@ RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNT COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc +COPY scripts /scripts + RUN mkdir -p /build_info/packages COPY udfclient_deps/packages /build_info/packages/udfclient_deps -RUN apt-get update - -RUN apt-get update && \ - apt-get -y install $(cat /build_info/packages/udfclient_deps/apt_get_packages) && \ - locale-gen en_US.UTF-8 && \ - update-locale LC_ALL=en_US.UTF-8 && \ - apt-get -y clean && \ - apt-get -y autoremove && \ - ldconfig +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages RUN addgroup --gid 1000 exasolution RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" From 4f1f1e970d65125b2a114c68057a1bc2a73a3d59 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 13:45:38 +0200 Subject: [PATCH 15/17] Correct path in Dockerfiles --- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index a0e27f2d9..cc27da4d5 100644 --- a/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -3,7 +3,7 @@ FROM {{ language_deps }} RUN mkdir -p /build_info/packages/flavor_base_deps COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions --allow-no-version +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions --allow-no-version COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile index a0e27f2d9..cc27da4d5 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/flavor_base_deps/Dockerfile @@ -3,7 +3,7 @@ FROM {{ language_deps }} RUN mkdir -p /build_info/packages/flavor_base_deps COPY flavor_base_deps/packages/apt_get_packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions --allow-no-version +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions --allow-no-version COPY flavor_base_deps/packages/python3_pip_packages /build_info/packages/flavor_base_deps RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --allow-no-version From 398f680681aeeb44eb345453aada357c17c83dd9 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 14:00:05 +0200 Subject: [PATCH 16/17] Add missing scripts dir to build_steps.py for cuda flavor --- .../flavor_base/build_steps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py index 0bf07a278..984ed9dde 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py @@ -21,7 +21,8 @@ def get_build_step(self) -> str: return "udfclient_deps" def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"01_nodoc": "ext/01_nodoc"} + return {"01_nodoc": "ext/01_nodoc" + "scripts": "ext/scripts"} def requires_tasks(self): return {"cuda_deps": AnalyzeCudaDeps} From e2e36afd0de16a06f38e97f03d770b55d6eacadc Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 23 Aug 2021 15:20:47 +0200 Subject: [PATCH 17/17] Fix syntax error --- .../flavor_base/build_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py index 984ed9dde..a99083042 100644 --- a/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py +++ b/flavors/python-3.6-data-science-cuda-EXASOL-6.2.0/flavor_base/build_steps.py @@ -21,7 +21,7 @@ def get_build_step(self) -> str: return "udfclient_deps" def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"01_nodoc": "ext/01_nodoc" + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} def requires_tasks(self):