Update Python to 3.10.13 (#239)

Updates to support latest security patches for Python 3.10. Manually fixed broken dependency download links for tcl/tk and ncurses.

Signed-off-by: Steve Pham <[email protected]>
Co-authored-by: Alex Peterson <[email protected]>

.github/workflows/build-container.yaml

@@ -40,7 +40,7 @@ jobs:
             ## Install build dependancies from apt
             apt-get update
             apt-get install -y build-essential curl git libssl-dev libffi-dev libbz2-dev libgdbm-compat-dev libgdbm-dev liblzma-dev libreadline-dev libtool \
-              ninja-build python3 python3-pip tcl8.6-dev tk8.6-dev texinfo software-properties-common wget
+              ninja-build python3 python3-pip tcl8.6-dev tk8.6-dev texinfo software-properties-common wget libc6
             python3 -m pip install boto3 certifi
             # Install Clang/GCC at specific version
             apt-get install -y clang-${CLANG_VER} gcc-${GCC_VER} g++-${GCC_VER}

package-system/python/Dockerfile.linux → package-system/python/Dockerfile

@@ -5,7 +5,6 @@
 # SPDX-License-Identifier: Apache-2.0 OR MIT
 #
 
-
 # The cpu architecture to base the docker base script from
 ARG INPUT_ARCHITECTURE=amd64
 
@@ -39,7 +38,7 @@ RUN apt-get clean && apt-get update
 # Setup time zone and locale data (necessary for SSL and HTTPS packages)
 RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata locales keyboard-configuration
 
-# Install the development packages needed to build AWS Gamelift
+# Install the development packages needed to build python
 RUN apt-get install -y autoconf \
                        build-essential \
                        cmake \
@@ -50,6 +49,7 @@ RUN apt-get install -y autoconf \
                        liblzma-dev \
                        libreadline-dev \
                        libtool \
+                       python3-dev \
                        python3 \
                        tcl8.6-dev \
                        tk8.6-dev \

package-system/python/FindPython.cmake.linux

@@ -8,4 +8,4 @@
 
 # force this into config mode, so that it uses the config files instead of module files.
 set(Python_DIR ${CMAKE_CURRENT_LIST_DIR})
-find_package(Python 3.10.5 REQUIRED CONFIG)
+find_package(Python 3.10.13 REQUIRED CONFIG)

package-system/python/build-linux.sh

@@ -157,7 +157,7 @@ docker build --build-arg INPUT_DOCKER_BUILD_SCRIPT=${DOCKER_BUILD_SCRIPT}\
  --build-arg INPUT_ARCHITECTURE=${DOCKER_INPUT_ARCHITECTURE}\
  --build-arg INPUT_IMAGE=ubuntu:${UBUNTU_BASE}\
  --build-arg INPUT_DEPENDENT_PACKAGE_FOLDERS=\"${DEP_PACKAGES_DOCKER_FOLDERNAMES}\"\
- -f Dockerfile.linux -t ${DOCKER_IMAGE_NAME}:latest temp"
+ -f Dockerfile -t ${DOCKER_IMAGE_NAME}:latest temp"
 
 echo $CMD_DOCKER_BUILD
 eval $CMD_DOCKER_BUILD

package-system/python/build_config.json

@@ -1,8 +1,8 @@
 {
    "git_url":"https://github.com/python/cpython.git",
-   "git_tag":"v3.10.5",
+   "git_tag":"v3.10.13",
    "package_name":"python",
-   "package_version":"3.10.5-rev4",
+   "package_version":"3.10.13-rev1",
    "package_url":"https://python.org",
    "package_license":"PSF-2.0",
    "package_license_file":"LICENSE",
@@ -25,7 +25,8 @@
                     "./package-linux.sh"
                 ],
                 "custom_test_cmd": [
-                    "./test-linux.sh"
+                    "./test-linux.sh",
+                    "x86_64"
                 ]
             },
             "Linux-aarch64":{
@@ -35,15 +36,16 @@
                 ],
                "custom_build_cmd": [
                     "./build-linux.sh",
-                    "python_3_10_5",
+                    "python_3_10_13",
                     "20.04",
                     "aarch64"
                 ],
                 "custom_install_cmd": [
                     "./package-linux.sh"
                 ],
                 "custom_test_cmd": [
-                    "./test-linux.sh"
+                    "./test-linux.sh",
+                    "aarch64"
                 ]
             }
         }

package-system/python/darwin_x64/FindPython.cmake

@@ -24,7 +24,7 @@ endif()
 # Python_Development_FOUND - The platform we are cross compiling for can link to python
 # and a target called 3rdParty::Python that you can use to depend on
 set(PY_BASE_DIR ${CMAKE_CURRENT_LIST_DIR}/Python.framework/Versions/3.10)
-set(${MY}_VERSION 3.10.5)
+set(${MY}_VERSION 3.10.13)
 set(${MY}_INTERPRETER_ID    "Python")
 set(${MY}_EXECUTABLE        ${PY_BASE_DIR}/bin/python3)
 set(${MY}_HOME              ${PY_BASE_DIR})

package-system/python/darwin_x64/PackageInfo.json

@@ -1,5 +1,5 @@
 {
-    "PackageName" : "python-3.10.5-rev2-darwin",
+    "PackageName" : "python-3.10.13-rev1-darwin",
     "License"     : "PSF-2.0",
     "URL"         : "https://python.org",
     "LicenseFile" : "LICENSE"

package-system/python/darwin_x64/make-python.sh

@@ -59,9 +59,9 @@ cd temp
 mkdir $SCRIPT_DIR/package
 
 echo ""
-echo "---------------- Cloning python 3.10.5 from git ----------------"
+echo "---------------- Cloning python 3.10.13 from git ----------------"
 echo ""
-git clone https://github.com/python/cpython.git --branch "v3.10.5" --depth 1
+git clone https://github.com/python/cpython.git --branch "v3.10.13" --depth 1
 retVal=$?
 if [ $retVal -ne 0 ]; then
     echo "Error cloning python from https://github.com/python/cpython.git"
@@ -106,6 +106,7 @@ echo "---------------- Installing spinx documentation tool into the v-env ------
 echo ""
 $VENV_BIN_DIR/python3 -m pip install sphinx
 
+
 cd $RELOC_SRC_DIR
 
 echo ""
@@ -167,8 +168,8 @@ cd $RELOC_SRC_DIR
 echo ""
 echo "---------------- Altering the produced framework folder to be relocatable ----------------"
 echo ""
-echo $VENV_BIN_DIR/python3 ./make_relocatable_python_framework.py --install-wheel --upgrade-pip --python-version 3.10.5 --use-existing-framework $FRAMEWORK_OUTPUT_FOLDER/Python.framework
-$VENV_BIN_DIR/python3 ./make_relocatable_python_framework.py --install-wheel --upgrade-pip --python-version 3.10.5 --use-existing-framework $FRAMEWORK_OUTPUT_FOLDER/Python.framework
+echo $VENV_BIN_DIR/python3 ./make_relocatable_python_framework.py --install-wheel --upgrade-pip --python-version 3.10.13 --use-existing-framework $FRAMEWORK_OUTPUT_FOLDER/Python.framework
+$VENV_BIN_DIR/python3 ./make_relocatable_python_framework.py --install-wheel --upgrade-pip --python-version 3.10.13 --use-existing-framework $FRAMEWORK_OUTPUT_FOLDER/Python.framework
 retVal=$?
 if [ $retVal -ne 0 ]; then
     echo "Could not make python relocatable!"
@@ -209,9 +210,6 @@ echo ""
 echo "---------------- Removing pip references from ensurepip ----------------"
 echo ""
 rm -f $SCRIPT_DIR/package/Python.framework/Versions/3.10/lib/python3.10/ensurepip/_bundled/pip-20*.whl
-cat $SCRIPT_DIR/package/Python.framework/Versions/3.10/lib/python3.10/ensurepip/__init__.py | sed 's/"20.1.1"/"22.0.3"/g' | sed 's/("pip", _PIP_VERSION, "py2.py3"),//g' > $SCRIPT_DIR/package/python/lib/python3.10/ensurepip/__init__.py_temp
-rm $SCRIPT_DIR/package/Python.framework/Versions/3.10/lib/python3.10/ensurepip/__init__.py
-mv $SCRIPT_DIR/package/Python.framework/Versions/3.10/lib/python3.10/ensurepip/__init__.py_temp $SCRIPT_DIR/package/python/lib/python3.10/ensurepip/__init__.py
 
 echo ""
 echo "----------------  Cleaning temp folder ----------------"

package-system/python/darwin_x64/open3d_python.patch

@@ -1,8 +1,21 @@
 diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
-index 4308a20..ec54cfb 100755
+index 8f33537..dad3d0c 100755
 --- a/Mac/BuildScript/build-installer.py
 +++ b/Mac/BuildScript/build-installer.py
-@@ -274,7 +274,7 @@ def library_recipes():
+@@ -264,17 +264,17 @@ def library_recipes():
+             tk_patches = ['tk868_on_10_8_10_9.patch']
+
+         else:
+-            tcl_tk_ver='8.6.12'
+-            tcl_checksum='87ea890821d2221f2ab5157bc5eb885f'
++            tcl_tk_ver='8.6.13'
++            tcl_checksum='0e4358aade2f5db8a8b6f2f6d9481ec2'
+
+-            tk_checksum='1d6dcf6120356e3d211e056dff5e462a'
++            tk_checksum='95adc33d55a133ee29bc9f81efdf31b2'
+             tk_patches = [ ]
+
+
          result.extend([
            dict(
                name="Tcl %s"%(tcl_tk_ver,),
@@ -28,7 +41,16 @@ index 4308a20..ec54cfb 100755
                ]
                ),
          ])
-@@ -1005,19 +1006,29 @@ def buildRecipe(recipe, basedir, archList):
+@@ -347,7 +348,7 @@ def library_recipes():
+                   "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),),
+               ],
+               patchscripts=[
+-                  ("ftp://ftp.invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2",
++                  ("https://src.fedoraproject.org/repo/pkgs/ncurses/ncurses-5.9-20120616-patch.sh.bz2/f54bf02a349f96a7c4f0d00922f3a0d4/ncurses-5.9-20120616-patch.sh.bz2",
+                    "f54bf02a349f96a7c4f0d00922f3a0d4"),
+                    ],
+               useLDFlags=False,
+@@ -1011,19 +1012,29 @@ def buildRecipe(recipe, basedir, archList):
                  configure_args.remove('--disable-shared')
              configure_args.extend(args)
 
@@ -70,7 +92,7 @@ index 4308a20..ec54cfb 100755
          else:
              configure_args.extend([
                  "CFLAGS=%s-mmacosx-version-min=%s -arch %s "
-@@ -1540,8 +1551,9 @@ def buildInstaller():
+@@ -1546,8 +1557,9 @@ def buildInstaller():
      pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents')
      pkgcontents = os.path.join(pkgroot, 'Packages')
      os.makedirs(pkgcontents)
@@ -81,4 +103,39 @@ index 4308a20..ec54cfb 100755
 +    return
 
      rsrcDir = os.path.join(pkgroot, 'Resources')
-
+
+diff --git a/Doc/Makefile b/Doc/Makefile
+index 4188f88..324acc4 100644
+--- a/Doc/Makefile
++++ b/Doc/Makefile
+@@ -12,7 +12,7 @@ JOBS         = auto
+ PAPER        =
+ SOURCES      =
+ DISTVERSION  = $(shell $(PYTHON) tools/extensions/patchlevel.py)
+-SPHINXERRORHANDLING = -W
++SPHINXERRORHANDLING =
+
+ # Internal variables.
+ PAPEROPT_a4     = -D latex_elements.papersize=a4paper
+diff --git a/Doc/requirements.txt b/Doc/requirements.txt
+index f43ce2c..2578393 100644
+--- a/Doc/requirements.txt
++++ b/Doc/requirements.txt
+@@ -3,13 +3,13 @@
+ # Sphinx version is pinned so that new versions that introduce new warnings
+ # won't suddenly cause build failures. Updating the version is fine as long
+ # as no warnings are raised by doing so.
+-sphinx==3.4.3
++sphinx
+ # Docutils version is pinned to a version compatible with Sphinx
+ # version <3.5.4. It can be removed after bumping Sphinx version to at
+ # least 3.5.4.
+-docutils==0.17.1
++docutils
+ # Jinja version is pinned to a version compatible with Sphinx version <4.5.
+-jinja2==3.0.3
++jinja2
+
+ blurb
+
+

package-system/python/linux_x64/FindPython.cmake

@@ -8,4 +8,4 @@
 
 # force this into config mode, so that it uses the config files instead of module files.
 set(Python_DIR ${CMAKE_CURRENT_LIST_DIR})
-find_package(Python 3.10.5 REQUIRED CONFIG)
+find_package(Python 3.10.13 REQUIRED CONFIG)

package-system/python/linux_x64/PackageInfo.json

@@ -1,5 +1,5 @@
 {
-    "PackageName" : "python-3.10.5-rev2-linux",
+    "PackageName" : "python-3.10.13-rev1-linux",
     "License"     : "PSF-2.0",
     "URL"         : "https://python.org",
     "LicenseFile" : "python/LICENSE"

package-system/python/linux_x64/make-python.sh

@@ -47,10 +47,10 @@ mkdir -p temp
 
 
 echo ""
-echo "--------------- Cloning python 3.10.5 from git ---------------"
+echo "--------------- Cloning python 3.10.13 from git ---------------"
 echo ""
 cd temp
-git clone https://github.com/python/cpython.git --branch v3.10.5 --depth 1
+git clone https://github.com/python/cpython.git --branch v3.10.13 --depth 1
 
 if [[ ! -d "cpython" ]]; then
     echo "Was unable to create cpython dir via git clone.  Is git installed?"
@@ -108,9 +108,9 @@ popd
 
 
 echo ""
-echo "--------------- Cloning openssl 1.1.1q and building it externally ---------------"
+echo "--------------- Cloning openssl 1.1.1w and building it externally ---------------"
 echo ""
-git clone https://github.com/openssl/openssl.git --branch "OpenSSL_1_1_1q" --depth 1
+git clone https://github.com/openssl/openssl.git --branch "OpenSSL_1_1_1w" --depth 1
 if [[ ! -d "openssl" ]]; then
     echo "Was unable to create openssl dir via git clone."
     exit 1
@@ -174,7 +174,7 @@ fi
 # Prepare the package folder
 cd $SCRIPT_DIR
 
-# Install the newly built python 3.10.5 to the package/python folder
+# Install the newly built python 3.10.13 to the package/python folder
 cd $SCRIPT_DIR
 cd temp
 cd cpython

package-system/python/linux_x64/python-config-version.cmake

@@ -9,7 +9,7 @@
 # this file is called to make sure that if we request a specific version
 # we respond only to that version
 
-set(PACKAGE_VERSION 3.10.5)
+set(PACKAGE_VERSION 3.10.13)
 set(PACKAGE_VERSION_EXACT False)
 set(PACKAGE_VERSION_COMPATIBLE False)
 
@@ -25,7 +25,7 @@ if (PACKAGE_FIND_VERSION_COUNT GREATER 1 AND NOT PACKAGE_FIND_VERSION_MINOR EQUA
     return()
 endif()
 
-if (PACKAGE_FIND_VERSION_COUNT GREATER 2 AND NOT PACKAGE_FIND_VERSION_PATCH EQUAL 5)
+if (PACKAGE_FIND_VERSION_COUNT GREATER 2 AND NOT PACKAGE_FIND_VERSION_PATCH EQUAL 13)
     return()
 endif()
 

package-system/python/linux_x64/python-config.cmake

@@ -24,7 +24,7 @@ endif()
 # Python_Development_FOUND - The platform we are cross compiling for can link to python
 # and a target called 3rdParty::Python that you can use to depend on
 
-set(${MY}_VERSION 3.10.5)
+set(${MY}_VERSION 3.10.13)
 set(${MY}_INTERPRETER_ID    "Python")
 set(${MY}_EXECUTABLE        ${CMAKE_CURRENT_LIST_DIR}/python/bin/python)
 set(${MY}_HOME              ${CMAKE_CURRENT_LIST_DIR}/python)

package-system/python/quick_validate_python.py

@@ -8,6 +8,8 @@
 
 # this script is run on built python executables to make sure they function.
 
+print("Simple import validation started")
+
 import sys
 
 try:

package-system/python/test-linux.sh

@@ -5,6 +5,34 @@
 #
 # SPDX-License-Identifier: Apache-2.0 OR MIT
 
-temp/build/python/bin/python3 quick_validate_python.py
 
-exit $?
+# Only run tests for packages on the same architecture
+TARGET_ARCH=$1
+CURRENT_HOST_ARCH=$(uname -m)
+
+if [ "${CURRENT_HOST_ARCH}" != "${TARGET_ARCH}" ]
+then
+    echo Cannot run the test for a ${TARGET_ARCH} on the current ${CURRENT_HOST_ARCH} architecture. Skipping test.
+    exit 0
+fi
+
+echo "Testing python"
+
+
+echo temp/build/python/bin/python3 --version 
+temp/build/python/bin/python3 --version 2>&1
+if [ $? -ne 0 ]
+then
+    echo "Error running validating python interpreter version"
+    exit 1
+fi
+
+echo temp/build/python/bin/python3 quick_validate_python.py
+temp/build/python/bin/python3 quick_validate_python.py 2>&1
+if [ $? -ne 0 ]
+then
+    echo "Error running the python interpreter against quick_validate_python.py"
+    exit 1
+fi
+
+exit 0

package-system/python/win_x64/FindPython.cmake

@@ -8,4 +8,4 @@
 
 # force this into config mode, so that it uses the config files instead of module files.
 set(Python_DIR ${CMAKE_CURRENT_LIST_DIR})
-find_package(Python 3.10.5 REQUIRED CONFIG)
+find_package(Python 3.10.13 REQUIRED CONFIG)

package-system/python/win_x64/PackageInfo.json

@@ -1,5 +1,5 @@
 {
-    "PackageName" : "python-3.10.5-rev1-windows",
+    "PackageName" : "python-3.10.13-rev1-windows",
     "URL"         : "https://python.org",
     "License"     : "PSF-2.0",
     "LicenseFile" : "python/LICENSE.txt"