From ebe71de8444d33a946820410022b5363260d856d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 10:06:43 -0500 Subject: [PATCH 001/161] test build with zig --- .github/workflows/build_python.yml | 32 ++++++ scripts/build_linux_zig.sh | 168 +++++++++++++++++++++++++++++ zigshim/zig_ar | 2 + zigshim/zig_cc | 2 + zigshim/zig_cxx | 2 + 5 files changed, 206 insertions(+) create mode 100755 scripts/build_linux_zig.sh create mode 100755 zigshim/zig_ar create mode 100755 zigshim/zig_cc create mode 100755 zigshim/zig_cxx diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index e1c8a6d9..f39e573b 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -17,7 +17,37 @@ env: platform_map: '{"x86_64": "linux/amd64", "aarch64": "linux/arm64/v8", "armv7l": "linux/arm/v7"}' jobs: + build_linux_zig: + name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arch: [ x86_64 ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build + run: | + /work/scripts/build_linux_zig.sh ${{ matrix.arch }} ${{ inputs.python_version }} + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: python + path: ./python*.zip + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: build-python + path: ./*python*.tar.gz + build_linux: + if: false name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} runs-on: ubuntu-latest strategy: @@ -89,6 +119,7 @@ jobs: # ./bin/python -m test || true build_windows: + if: false name: Windows ${{ inputs.python_version }} x86_64 (build) runs-on: windows-latest @@ -148,6 +179,7 @@ jobs: # ./bin/python -m test || true build_macos: + if: false name: MacOS ${{ inputs.python_version }} universal2 (build) runs-on: macos-latest diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh new file mode 100755 index 00000000..a860f5de --- /dev/null +++ b/scripts/build_linux_zig.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +ARCH=$1 +PYTHON_FULL_VER=$2 +PYTHON_VER=$(echo ${PYTHON_FULL_VER} | cut -d "." -f 1-2) + +set -ex + +WORKDIR=$(pwd) +BUILDDIR=${WORKDIR}/build +DEPSDIR=${WORKDIR}/deps + +apt update +apt -y upgrade +apt -y install wget build-essential cmake lld + +cd / +wget https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz +tar -xf zig*.tar.xz +cd ${WORKDIR} + +cp -r zigshim/* /zig-linux-x86_64-0.11.0 +export PATH=${PATH}:/zig-linux-x86_64-0.11.0 + +mkdir ${BUILDDIR} +mkdir ${DEPSDIR} +cd ${BUILDDIR} + +export AR=zig_ar +export CC=zig_cc +export CXX=zig_cxx +#export LD=lld + +export TARGET=${ARCH}-linux-gnu.2.17 +export ZIG_TARGET=${TARGET} + +wget https://zlib.net/fossils/zlib-1.3.tar.gz +tar -xf zlib*.tar.gz +cd zlib* +./configure --prefix=${DEPSDIR} +make -j4 +make install +cd ${BUILDDIR} + +wget https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar.gz +tar -xf OpenSSL*.tar.gz +cd openssl-OpenSSL* +./config no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} +make -j4 +make install_sw +cd ${BUILDDIR} + +wget https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz +tar -xf libffi*.tar.gz +cd libffi* +./configure --prefix=${DEPSDIR} +make -j4 +make install +cd ${BUILDDIR} + +wget https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz +tar -xf sqlite*.tar.gz +cd sqlite* +./configure --prefix=${DEPSDIR} +make -j4 +make install +cd ${BUILDDIR} + +wget https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz +tar -xf readline*.tar.gz +cd readline* +./configure --prefix=${DEPSDIR} +make -j4 +make install +cd ${BUILDDIR} + +wget https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz +tar -xf ncurses*.tar.gz +cd ncurses* +./configure --prefix=${DEPSDIR} +make -j4 +make install +cd ${BUILDDIR} + +wget -O bzip2.tar.gz https://github.com/commontk/bzip2/tarball/master +tar -xf bzip2*.tar.gz +cd commontk-bzip2* +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. +make -j4 +make install +cd ${BUILDDIR} + +wget https://github.com/tukaani-project/xz/releases/download/v5.4.5/xz-5.4.5.tar.gz +tar -xf xz*.tar.gz +cd xz* +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. +make -j4 +make install +cd ${BUILDDIR} + +wget -O libuuid-cmake.tar.gz https://github.com/gershnik/libuuid-cmake/archive/refs/tags/v2.39.3.tar.gz +tar -xf libuuid*tar.gz +cd libuuid* +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. +make -j4 +make install +cd ${BUILDDIR} + +wget https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz +tar -xf gdbm*.tar.gz +cd gdbm* +./configure --prefix=${DEPSDIR} +make -j4 +make install +cd ${BUILDDIR} + +wget https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz +tar -xf tcl*.tar.gz +cd tcl*/unix +./configure --prefix=${DEPSDIR} +make -j4 +make install +cd ${BUILDDIR} + +export CFLAGS="-I${DEPSDIR}/include" +wget -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/python3.10 +tar -xf python-cmake-buildsystem.tar.gz +rm python-cmake-buildsystem.tar.gz +mv *python-cmake-buildsystem* python-cmake-buildsystem +mkdir python-build +mkdir python-install +cd python-build +cmake \ + -DCMAKE_C_STANDARD=99 \ + -DPYTHON_VERSION=${PYTHON_FULL_VER} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=/build/python-install \ + -DBUILD_EXTENSIONS_AS_BUILTIN=OFF \ + -DBUILD_LIBPYTHON_SHARED=ON \ + -DBUILD_TESTING=ON \ + -DINSTALL_TEST=OFF \ + -DINSTALL_MANUAL=OFF \ + -DOPENSSL_ROOT_DIR:PATH=${DEPSDIR} \ + -DSQLite3_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DSQLite3_LIBRARY:FILEPATH=${DEPSDIR}/lib/libsqlite3.a \ + -DZLIB_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DZLIB_LIBRARY:FILEPATH=${DEPSDIR}/lib/libz.a \ + -DLZMA_INCLUDE_PATH:PATH=${DEPSDIR}/include \ + -DLZMA_LIBRARY:FILEPATH=${DEPSDIR}/lib/liblzma.a \ + -DBZIP2_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DBZIP2_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libbz2.a \ + -DLibFFI_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DLibFFI_LIBRARY:FILEPATH=${DEPSDIR}/lib/libffi.a \ + -DREADLINE_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/readline/readline.h \ + -DREADLINE_LIBRARY:FILEPATH=${DEPSDIR}/lib/libreadline.a \ + -DUUID_LIBRARY:FILEPATH=${DEPSDIR}/lib/libuuid_static.a \ + -DCURSES_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libncurses.a \ + -DPANEL_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libpanel.a \ + -DGDBM_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm.a \ + ../python-cmake-buildsystem +make -j4 +make install diff --git a/zigshim/zig_ar b/zigshim/zig_ar new file mode 100755 index 00000000..80b2e84f --- /dev/null +++ b/zigshim/zig_ar @@ -0,0 +1,2 @@ +#!/bin/bash +zig ar "$@" \ No newline at end of file diff --git a/zigshim/zig_cc b/zigshim/zig_cc new file mode 100755 index 00000000..5e5302c2 --- /dev/null +++ b/zigshim/zig_cc @@ -0,0 +1,2 @@ +#!/bin/bash +zig cc -target $ZIG_TARGET "$@" \ No newline at end of file diff --git a/zigshim/zig_cxx b/zigshim/zig_cxx new file mode 100755 index 00000000..15fd2e9f --- /dev/null +++ b/zigshim/zig_cxx @@ -0,0 +1,2 @@ +#!/bin/bash +zig c++ -target $ZIG_TARGET "$@" \ No newline at end of file From 3b27e1d744a6de08b1a4ca2c6853ab5e8ad4e3d1 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 10:07:39 -0500 Subject: [PATCH 002/161] use correct script path --- .github/workflows/build_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index f39e573b..6b17513f 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -31,7 +31,7 @@ jobs: - name: Build run: | - /work/scripts/build_linux_zig.sh ${{ matrix.arch }} ${{ inputs.python_version }} + ./scripts/build_linux_zig.sh ${{ matrix.arch }} ${{ inputs.python_version }} - name: Upload artifacts uses: actions/upload-artifact@v3 From 14becbec45aca2f231a9f8ab0c7edcef0477f016 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 10:08:17 -0500 Subject: [PATCH 003/161] use sudo --- .github/workflows/build_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 6b17513f..2716764c 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -31,7 +31,7 @@ jobs: - name: Build run: | - ./scripts/build_linux_zig.sh ${{ matrix.arch }} ${{ inputs.python_version }} + sudo ./scripts/build_linux_zig.sh ${{ matrix.arch }} ${{ inputs.python_version }} - name: Upload artifacts uses: actions/upload-artifact@v3 From 0b81d74780454c5b5fe53b062eaafe3e0ffb9a69 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 10:12:49 -0500 Subject: [PATCH 004/161] remove tarballs --- scripts/build_linux_zig.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index a860f5de..97c24c71 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -36,6 +36,7 @@ export ZIG_TARGET=${TARGET} wget https://zlib.net/fossils/zlib-1.3.tar.gz tar -xf zlib*.tar.gz +rm *.tar.gz cd zlib* ./configure --prefix=${DEPSDIR} make -j4 @@ -44,6 +45,7 @@ cd ${BUILDDIR} wget https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar.gz tar -xf OpenSSL*.tar.gz +rm *.tar.gz cd openssl-OpenSSL* ./config no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} make -j4 @@ -52,6 +54,7 @@ cd ${BUILDDIR} wget https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz tar -xf libffi*.tar.gz +rm *.tar.gz cd libffi* ./configure --prefix=${DEPSDIR} make -j4 @@ -60,6 +63,7 @@ cd ${BUILDDIR} wget https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz tar -xf sqlite*.tar.gz +rm *.tar.gz cd sqlite* ./configure --prefix=${DEPSDIR} make -j4 @@ -68,6 +72,7 @@ cd ${BUILDDIR} wget https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz tar -xf readline*.tar.gz +rm *.tar.gz cd readline* ./configure --prefix=${DEPSDIR} make -j4 @@ -76,6 +81,7 @@ cd ${BUILDDIR} wget https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz +rm *.tar.gz cd ncurses* ./configure --prefix=${DEPSDIR} make -j4 @@ -84,6 +90,7 @@ cd ${BUILDDIR} wget -O bzip2.tar.gz https://github.com/commontk/bzip2/tarball/master tar -xf bzip2*.tar.gz +rm *.tar.gz cd commontk-bzip2* mkdir build cd build @@ -94,6 +101,7 @@ cd ${BUILDDIR} wget https://github.com/tukaani-project/xz/releases/download/v5.4.5/xz-5.4.5.tar.gz tar -xf xz*.tar.gz +rm *.tar.gz cd xz* mkdir build cd build @@ -104,6 +112,7 @@ cd ${BUILDDIR} wget -O libuuid-cmake.tar.gz https://github.com/gershnik/libuuid-cmake/archive/refs/tags/v2.39.3.tar.gz tar -xf libuuid*tar.gz +rm *.tar.gz cd libuuid* mkdir build cd build @@ -114,6 +123,7 @@ cd ${BUILDDIR} wget https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz tar -xf gdbm*.tar.gz +rm *.tar.gz cd gdbm* ./configure --prefix=${DEPSDIR} make -j4 @@ -122,6 +132,7 @@ cd ${BUILDDIR} wget https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz tar -xf tcl*.tar.gz +rm *.tar.gz cd tcl*/unix ./configure --prefix=${DEPSDIR} make -j4 @@ -131,7 +142,7 @@ cd ${BUILDDIR} export CFLAGS="-I${DEPSDIR}/include" wget -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/python3.10 tar -xf python-cmake-buildsystem.tar.gz -rm python-cmake-buildsystem.tar.gz +rm *.tar.gz mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install From f548449ada6571503364a39688b846c3a8efc833 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 10:13:34 -0500 Subject: [PATCH 005/161] skip apt upgrade --- scripts/build_linux_zig.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 97c24c71..1bc51d05 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -11,7 +11,6 @@ BUILDDIR=${WORKDIR}/build DEPSDIR=${WORKDIR}/deps apt update -apt -y upgrade apt -y install wget build-essential cmake lld cd / From 66583deca712f4c0db69e9dec8f000b263ab99e1 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 10:14:51 -0500 Subject: [PATCH 006/161] quiet wget --- scripts/build_linux_zig.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 1bc51d05..0581e611 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -14,7 +14,7 @@ apt update apt -y install wget build-essential cmake lld cd / -wget https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz +wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz tar -xf zig*.tar.xz cd ${WORKDIR} @@ -33,7 +33,7 @@ export CXX=zig_cxx export TARGET=${ARCH}-linux-gnu.2.17 export ZIG_TARGET=${TARGET} -wget https://zlib.net/fossils/zlib-1.3.tar.gz +wget -q https://zlib.net/fossils/zlib-1.3.tar.gz tar -xf zlib*.tar.gz rm *.tar.gz cd zlib* @@ -42,7 +42,7 @@ make -j4 make install cd ${BUILDDIR} -wget https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar.gz +wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar.gz tar -xf OpenSSL*.tar.gz rm *.tar.gz cd openssl-OpenSSL* @@ -51,7 +51,7 @@ make -j4 make install_sw cd ${BUILDDIR} -wget https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz +wget -q https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz tar -xf libffi*.tar.gz rm *.tar.gz cd libffi* @@ -60,7 +60,7 @@ make -j4 make install cd ${BUILDDIR} -wget https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz +wget -q https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz tar -xf sqlite*.tar.gz rm *.tar.gz cd sqlite* @@ -69,7 +69,7 @@ make -j4 make install cd ${BUILDDIR} -wget https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz +wget -q https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz tar -xf readline*.tar.gz rm *.tar.gz cd readline* @@ -78,7 +78,7 @@ make -j4 make install cd ${BUILDDIR} -wget https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz +wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* @@ -87,7 +87,7 @@ make -j4 make install cd ${BUILDDIR} -wget -O bzip2.tar.gz https://github.com/commontk/bzip2/tarball/master +wget -q -O bzip2.tar.gz https://github.com/commontk/bzip2/tarball/master tar -xf bzip2*.tar.gz rm *.tar.gz cd commontk-bzip2* @@ -98,7 +98,7 @@ make -j4 make install cd ${BUILDDIR} -wget https://github.com/tukaani-project/xz/releases/download/v5.4.5/xz-5.4.5.tar.gz +wget -q https://github.com/tukaani-project/xz/releases/download/v5.4.5/xz-5.4.5.tar.gz tar -xf xz*.tar.gz rm *.tar.gz cd xz* @@ -109,7 +109,7 @@ make -j4 make install cd ${BUILDDIR} -wget -O libuuid-cmake.tar.gz https://github.com/gershnik/libuuid-cmake/archive/refs/tags/v2.39.3.tar.gz +wget -q -O libuuid-cmake.tar.gz https://github.com/gershnik/libuuid-cmake/archive/refs/tags/v2.39.3.tar.gz tar -xf libuuid*tar.gz rm *.tar.gz cd libuuid* @@ -120,7 +120,7 @@ make -j4 make install cd ${BUILDDIR} -wget https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz +wget -q https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz tar -xf gdbm*.tar.gz rm *.tar.gz cd gdbm* @@ -129,7 +129,7 @@ make -j4 make install cd ${BUILDDIR} -wget https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz +wget -q https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz tar -xf tcl*.tar.gz rm *.tar.gz cd tcl*/unix @@ -139,7 +139,7 @@ make install cd ${BUILDDIR} export CFLAGS="-I${DEPSDIR}/include" -wget -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/python3.10 +wget -q -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/python3.10 tar -xf python-cmake-buildsystem.tar.gz rm *.tar.gz mv *python-cmake-buildsystem* python-cmake-buildsystem From 4b5f8074a25fbb2445d4fd0d30fe7722aa2c9175 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 10:55:39 -0500 Subject: [PATCH 007/161] switch all to fork --- scripts/build_linux.sh | 3 +-- scripts/build_linux_zig.sh | 2 +- scripts/build_macos.sh | 3 +-- scripts/build_windows.sh | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh index 87b31328..64d65867 100755 --- a/scripts/build_linux.sh +++ b/scripts/build_linux.sh @@ -60,8 +60,7 @@ cd /build mkdir python-build mkdir python-install -git clone https://github.com/bjia56/python-cmake-buildsystem.git --branch python3.10 --single-branch --depth 1 -#git clone https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git +git clone https://github.com/bjia56/python-cmake-buildsystem.git --branch portable-python --single-branch --depth 1 echo "::endgroup::" ############# diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 0581e611..9d1d3f48 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -139,7 +139,7 @@ make install cd ${BUILDDIR} export CFLAGS="-I${DEPSDIR}/include" -wget -q -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/python3.10 +wget -q -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/portable-python tar -xf python-cmake-buildsystem.tar.gz rm *.tar.gz mv *python-cmake-buildsystem* python-cmake-buildsystem diff --git a/scripts/build_macos.sh b/scripts/build_macos.sh index f23d2fba..75d70535 100755 --- a/scripts/build_macos.sh +++ b/scripts/build_macos.sh @@ -20,8 +20,7 @@ mkdir deps export MACOSX_DEPLOYMENT_TARGET=10.5 -git clone https://github.com/bjia56/python-cmake-buildsystem.git --branch python3.10 --single-branch --depth 1 -#git clone https://github.com/bjia56/python-cmake-buildsystem.git --branch macos-arm64 --single-branch --depth 1 +git clone https://github.com/bjia56/python-cmake-buildsystem.git --branch portable-python --single-branch --depth 1 echo "::endgroup::" ########### diff --git a/scripts/build_windows.sh b/scripts/build_windows.sh index 31196580..7a38e8cc 100755 --- a/scripts/build_windows.sh +++ b/scripts/build_windows.sh @@ -16,8 +16,7 @@ mkdir python-build mkdir python-install mkdir deps -git clone https://github.com/bjia56/python-cmake-buildsystem.git --branch python3.10 --single-branch --depth 1 -#git clone https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git +git clone https://github.com/bjia56/python-cmake-buildsystem.git --branch portable-python --single-branch --depth 1 echo "::endgroup::" ########### From 9278247340b10a40d2e0c4928a94a995789fae87 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 12:46:41 -0500 Subject: [PATCH 008/161] update deps and compilation --- scripts/build_linux_zig.sh | 24 ++++++++++++++++++------ zigshim/zig_cc | 7 ++++++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 9d1d3f48..d98e49d6 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -11,7 +11,7 @@ BUILDDIR=${WORKDIR}/build DEPSDIR=${WORKDIR}/deps apt update -apt -y install wget build-essential cmake lld +apt -y install wget build-essential pkg-config cmake autoconf git python3 meson clang cd / wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz @@ -28,7 +28,6 @@ cd ${BUILDDIR} export AR=zig_ar export CC=zig_cc export CXX=zig_cxx -#export LD=lld export TARGET=${ARCH}-linux-gnu.2.17 export ZIG_TARGET=${TARGET} @@ -124,7 +123,7 @@ wget -q https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz tar -xf gdbm*.tar.gz rm *.tar.gz cd gdbm* -./configure --prefix=${DEPSDIR} +./configure --enable-libgdbm-compat --prefix=${DEPSDIR} make -j4 make install cd ${BUILDDIR} @@ -138,7 +137,15 @@ make -j4 make install cd ${BUILDDIR} -export CFLAGS="-I${DEPSDIR}/include" +wget -q https://prdownloads.sourceforge.net/tcl/tk8.6.13-src.tar.gz +tar -xf tk*.tar.gz +rm *.tar.gz +cd tk*/unix +CFLAGS="-I${DEPSDIR}/include" ./configure --prefix=${DEPSDIR} +make -j4 +make install +cd ${BUILDDIR} + wget -q -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/portable-python tar -xf python-cmake-buildsystem.tar.gz rm *.tar.gz @@ -146,17 +153,19 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -cmake \ +CFLAGS="-I${DEPSDIR}/include" cmake \ -DCMAKE_C_STANDARD=99 \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_INSTALL_PREFIX:PATH=/build/python-install \ -DBUILD_EXTENSIONS_AS_BUILTIN=OFF \ -DBUILD_LIBPYTHON_SHARED=ON \ + -DUSE_SYSTEM_LIBRARIES=OFF \ -DBUILD_TESTING=ON \ -DINSTALL_TEST=OFF \ -DINSTALL_MANUAL=OFF \ - -DOPENSSL_ROOT_DIR:PATH=${DEPSDIR} \ + -DOPENSSL_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DOPENSSL_LIBRARIES="${DEPSDIR}/lib/libssl.a;${DEPSDIR}/lib/libcrypto.a" \ -DSQLite3_INCLUDE_DIR:PATH=${DEPSDIR}/include \ -DSQLite3_LIBRARY:FILEPATH=${DEPSDIR}/lib/libsqlite3.a \ -DZLIB_INCLUDE_DIR:PATH=${DEPSDIR}/include \ @@ -172,7 +181,10 @@ cmake \ -DUUID_LIBRARY:FILEPATH=${DEPSDIR}/lib/libuuid_static.a \ -DCURSES_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libncurses.a \ -DPANEL_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libpanel.a \ + -DGDBM_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/gdbm.h \ -DGDBM_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm.a \ + -DGDBM_COMPAT_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm_compat.a \ + -DNDBM_TAG=NDBM \ ../python-cmake-buildsystem make -j4 make install diff --git a/zigshim/zig_cc b/zigshim/zig_cc index 5e5302c2..97a42a4f 100755 --- a/zigshim/zig_cc +++ b/zigshim/zig_cc @@ -1,2 +1,7 @@ #!/bin/bash -zig cc -target $ZIG_TARGET "$@" \ No newline at end of file +if [ "$1" == "-Wl,--version" ]; then + # hack for meson to detect ld correctly + clang "$@" +else + zig cc -target $ZIG_TARGET "$@" +fi \ No newline at end of file From 40fe1404c5c244dd78d418c2ad66618e49ae54f7 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 12:56:04 -0500 Subject: [PATCH 009/161] drop tcl/tk for now --- scripts/build_linux_zig.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index d98e49d6..bc2e8810 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -137,14 +137,14 @@ make -j4 make install cd ${BUILDDIR} -wget -q https://prdownloads.sourceforge.net/tcl/tk8.6.13-src.tar.gz -tar -xf tk*.tar.gz -rm *.tar.gz -cd tk*/unix -CFLAGS="-I${DEPSDIR}/include" ./configure --prefix=${DEPSDIR} -make -j4 -make install -cd ${BUILDDIR} +#wget -q https://prdownloads.sourceforge.net/tcl/tk8.6.13-src.tar.gz +#tar -xf tk*.tar.gz +#rm *.tar.gz +#cd tk*/unix +#CFLAGS="-I${DEPSDIR}/include" ./configure --prefix=${DEPSDIR} +#make -j4 +#make install +#cd ${BUILDDIR} wget -q -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/portable-python tar -xf python-cmake-buildsystem.tar.gz From f2ec0933234f48696ceb87a6a5f309da5cbad58d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 13:00:28 -0500 Subject: [PATCH 010/161] ncurses config settings --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index bc2e8810..ab524d69 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -81,7 +81,7 @@ wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* -./configure --prefix=${DEPSDIR} +./configure --with-normal --enable-overwrite --prefix=${DEPSDIR} make -j4 make install cd ${BUILDDIR} From 31c81c95c4b77d10db1d76a66ad8fe80642f03b4 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 14:08:57 -0500 Subject: [PATCH 011/161] add gha groups --- scripts/build_linux_zig.sh | 104 +++++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 10 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index ab524d69..0a91fa11 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -10,8 +10,14 @@ WORKDIR=$(pwd) BUILDDIR=${WORKDIR}/build DEPSDIR=${WORKDIR}/deps +######################## +# Install dependencies # +######################## +echo "::group::Install dependencies" + +export DEBIAN_FRONTEND=noninteractive apt update -apt -y install wget build-essential pkg-config cmake autoconf git python3 meson clang +apt -y install wget build-essential pkg-config cmake autoconf git python3 meson clang patchelf cd / wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz @@ -23,7 +29,6 @@ export PATH=${PATH}:/zig-linux-x86_64-0.11.0 mkdir ${BUILDDIR} mkdir ${DEPSDIR} -cd ${BUILDDIR} export AR=zig_ar export CC=zig_cc @@ -32,6 +37,13 @@ export CXX=zig_cxx export TARGET=${ARCH}-linux-gnu.2.17 export ZIG_TARGET=${TARGET} +echo "::endgroup::" +######## +# zlib # +######## +echo "::group::zlib" +cd ${BUILDDIR} + wget -q https://zlib.net/fossils/zlib-1.3.tar.gz tar -xf zlib*.tar.gz rm *.tar.gz @@ -39,6 +51,12 @@ cd zlib* ./configure --prefix=${DEPSDIR} make -j4 make install + +echo "::endgroup::" +########### +# OpenSSL # +########### +echo "::group::OpenSSL" cd ${BUILDDIR} wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar.gz @@ -48,6 +66,12 @@ cd openssl-OpenSSL* ./config no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} make -j4 make install_sw + +echo "::endgroup::" +########## +# libffi # +########## +echo "::group::libffi" cd ${BUILDDIR} wget -q https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz @@ -57,6 +81,12 @@ cd libffi* ./configure --prefix=${DEPSDIR} make -j4 make install + +echo "::endgroup::" +########### +# sqlite3 # +########### +echo "::group::sqlite3" cd ${BUILDDIR} wget -q https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz @@ -66,6 +96,12 @@ cd sqlite* ./configure --prefix=${DEPSDIR} make -j4 make install + +echo "::endgroup::" +############ +# readline # +############ +echo "::group::readline" cd ${BUILDDIR} wget -q https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz @@ -75,6 +111,12 @@ cd readline* ./configure --prefix=${DEPSDIR} make -j4 make install + +echo "::endgroup::" +########### +# ncurses # +########### +echo "::group::ncurses" cd ${BUILDDIR} wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz @@ -84,6 +126,12 @@ cd ncurses* ./configure --with-normal --enable-overwrite --prefix=${DEPSDIR} make -j4 make install + +echo "::endgroup::" +######### +# bzip2 # +######### +echo "::group::bzip2" cd ${BUILDDIR} wget -q -O bzip2.tar.gz https://github.com/commontk/bzip2/tarball/master @@ -95,6 +143,12 @@ cd build cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. make -j4 make install + +echo "::endgroup::" +###### +# xz # +###### +echo "::group::xz" cd ${BUILDDIR} wget -q https://github.com/tukaani-project/xz/releases/download/v5.4.5/xz-5.4.5.tar.gz @@ -106,6 +160,12 @@ cd build cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. make -j4 make install + +echo "::endgroup::" +######## +# uuid # +######## +echo "::group::uuid" cd ${BUILDDIR} wget -q -O libuuid-cmake.tar.gz https://github.com/gershnik/libuuid-cmake/archive/refs/tags/v2.39.3.tar.gz @@ -117,6 +177,12 @@ cd build cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. make -j4 make install + +echo "::endgroup::" +######## +# gdbm # +######## +echo "::group::gdbm" cd ${BUILDDIR} wget -q https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz @@ -126,15 +192,27 @@ cd gdbm* ./configure --enable-libgdbm-compat --prefix=${DEPSDIR} make -j4 make install + +echo "::endgroup::" +####### +# tcl # +####### +echo "::group::tcl" cd ${BUILDDIR} -wget -q https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz -tar -xf tcl*.tar.gz -rm *.tar.gz -cd tcl*/unix -./configure --prefix=${DEPSDIR} -make -j4 -make install +#wget -q https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz +#tar -xf tcl*.tar.gz +#rm *.tar.gz +#cd tcl*/unix +#./configure --prefix=${DEPSDIR} +#make -j4 +#make install + +echo "::endgroup::" +###### +# tk # +###### +echo "::group::tk" cd ${BUILDDIR} #wget -q https://prdownloads.sourceforge.net/tcl/tk8.6.13-src.tar.gz @@ -144,7 +222,13 @@ cd ${BUILDDIR} #CFLAGS="-I${DEPSDIR}/include" ./configure --prefix=${DEPSDIR} #make -j4 #make install -#cd ${BUILDDIR} + +echo "::endgroup::" +########## +# Python # +########## +echo "::group::Python" +cd ${BUILDDIR} wget -q -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/portable-python tar -xf python-cmake-buildsystem.tar.gz From fc4918e97aba85db393601c76d5533042a253208 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 14:11:59 -0500 Subject: [PATCH 012/161] copy over patching from old linux build --- scripts/build_linux_zig.sh | 78 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 0a91fa11..28df915f 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -272,3 +272,81 @@ CFLAGS="-I${DEPSDIR}/include" cmake \ ../python-cmake-buildsystem make -j4 make install + +tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz . + +echo "::endgroup::" +############################################# +# Check executable dependencies (pre-patch) # +############################################# +echo "::group::Check executable dependencies (pre-patch)" +cd ${BUILDDIR} + +cd python-install +echo "python dependencies" +ldd -v -r ./bin/python +echo +echo "libpython dependencies" +ldd -v -r ./lib/libpython${PYTHON_VER}.so + +echo "::endgroup::" +################ +# Patch python # +################ +echo "::group::Patch python" +cd ${BUILDDIR} + +cd python-install +${WORKDIR}/scripts/patch_libpython.sh ./lib/libpython${PYTHON_VER}.so ./bin/python +patchelf --replace-needed libpython${PYTHON_VER}.so "\$ORIGIN/../lib/libpython${PYTHON_VER}.so" ./bin/python + +echo "::endgroup::" +############################################## +# Check executable dependencies (post-patch) # +############################################## +echo "::group::Check executable dependencies (post-patch)" +cd ${BUILDDIR} + +cd python-install +# we don't make ldd errors fatal here since sometimes ldd can +# crash but the patched binaries still work +echo "python dependencies" +ldd -v -r ./bin/python || true +echo +echo "libpython dependencies" +ldd -v -r ./lib/libpython${PYTHON_VER}.so || true + +echo "::endgroup::" +############### +# Test python # +############### +echo "::group::Test python" +cd ${BUILDDIR} + +cd python-install +./bin/python --version + +echo "::endgroup::" +############### +# Preload pip # +############### +echo "::group::Preload pip" +cd ${BUILDDIR} + +cd python-install +./bin/python -m ensurepip + +echo "::endgroup::" +################### +# Compress output # +################### +echo "::group::Compress output" +cd ${BUILDDIR} + +python3 -m pip install pyclean +python3 -m pyclean -v python-install +mv python-install python-${PYTHON_FULL_VER}-linux-${ARCH} +tar -czf ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz python-${PYTHON_FULL_VER}-linux-${ARCH} +zip ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.zip $(tar tf ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz) + +echo "::endgroup::" \ No newline at end of file From a86ed6f3d3cb5032b654b4b289da8ac86b513197 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 14:23:21 -0500 Subject: [PATCH 013/161] fix install dir --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 28df915f..0cbdcb6a 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -241,7 +241,7 @@ CFLAGS="-I${DEPSDIR}/include" cmake \ -DCMAKE_C_STANDARD=99 \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=/build/python-install \ + -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ -DBUILD_EXTENSIONS_AS_BUILTIN=OFF \ -DBUILD_LIBPYTHON_SHARED=ON \ -DUSE_SYSTEM_LIBRARIES=OFF \ From 0a3275284062efbc6bac379d1f20a0abcd09699d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 14:57:20 -0500 Subject: [PATCH 014/161] skip 3.8 for now --- .github/workflows/build_python.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 2716764c..d66da62c 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -18,6 +18,7 @@ env: jobs: build_linux_zig: + if: ${{ !startsWith('3.8', inputs.python_version) }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} runs-on: ubuntu-latest strategy: @@ -47,7 +48,7 @@ jobs: path: ./*python*.tar.gz build_linux: - if: false + if: ${{ false && startsWith('3.8', inputs.python_version) }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} runs-on: ubuntu-latest strategy: From 73a08562e646200d748aad3826f1e830b81ebd9f Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 14:58:15 -0500 Subject: [PATCH 015/161] switch order of args --- .github/workflows/build_python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index d66da62c..ce3fb9b1 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -18,7 +18,7 @@ env: jobs: build_linux_zig: - if: ${{ !startsWith('3.8', inputs.python_version) }} + if: ${{ !startsWith(inputs.python_version, '3.8') }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} runs-on: ubuntu-latest strategy: @@ -48,7 +48,7 @@ jobs: path: ./*python*.tar.gz build_linux: - if: ${{ false && startsWith('3.8', inputs.python_version) }} + if: ${{ false && startsWith(inputs.python_version, '3.8') }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} runs-on: ubuntu-latest strategy: From 298fb8967e793d95f0e8388cc16c147da513dda8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:00:12 -0500 Subject: [PATCH 016/161] add aarch64 --- .github/workflows/build_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index ce3fb9b1..b55c671c 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - arch: [ x86_64 ] + arch: [ x86_64, aarch64 ] steps: - name: Checkout From c88da929fccdece6a9840847305aa550929d6620 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:03:56 -0500 Subject: [PATCH 017/161] set CHOST --- scripts/build_linux_zig.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 0cbdcb6a..1c1438bc 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -33,6 +33,7 @@ mkdir ${DEPSDIR} export AR=zig_ar export CC=zig_cc export CXX=zig_cxx +export CHOST=${ARCH} export TARGET=${ARCH}-linux-gnu.2.17 export ZIG_TARGET=${TARGET} From 153b66c3bd1302a0ee4ec3b42e9503cc0f4a7340 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:09:06 -0500 Subject: [PATCH 018/161] specify openssl target --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 1c1438bc..73c08d4d 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -64,7 +64,7 @@ wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar. tar -xf OpenSSL*.tar.gz rm *.tar.gz cd openssl-OpenSSL* -./config no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} +./config linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} make -j4 make install_sw From cafc3bbdc1d52f825c65c7bed7bfbf5647b136c6 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:10:42 -0500 Subject: [PATCH 019/161] use Configure --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 73c08d4d..0f9dac94 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -64,7 +64,7 @@ wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar. tar -xf OpenSSL*.tar.gz rm *.tar.gz cd openssl-OpenSSL* -./config linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} +./Configure linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} make -j4 make install_sw From 4f63e1f11d08fd224b109d8814aaa20056986da3 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:16:01 -0500 Subject: [PATCH 020/161] more cross compile flags --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 0f9dac94..5fed11d8 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -79,7 +79,7 @@ wget -q https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.t tar -xf libffi*.tar.gz rm *.tar.gz cd libffi* -./configure --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 702fe7b0b833fabbe6f5e998e89d4a2bff17c9e5 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:19:31 -0500 Subject: [PATCH 021/161] more cross compile flags --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 5fed11d8..cd5bdc85 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -94,7 +94,7 @@ wget -q https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz tar -xf sqlite*.tar.gz rm *.tar.gz cd sqlite* -./configure --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 676ed10e1aa308e90d85f05c60120ad780293c72 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:25:34 -0500 Subject: [PATCH 022/161] more cross compile flags --- scripts/build_linux_zig.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index cd5bdc85..f791234c 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -109,7 +109,7 @@ wget -q https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz tar -xf readline*.tar.gz rm *.tar.gz cd readline* -./configure --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -124,7 +124,7 @@ wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* -./configure --with-normal --enable-overwrite --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-normal --enable-overwrite --prefix=${DEPSDIR} make -j4 make install @@ -190,7 +190,7 @@ wget -q https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz tar -xf gdbm*.tar.gz rm *.tar.gz cd gdbm* -./configure --enable-libgdbm-compat --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --enable-libgdbm-compat --prefix=${DEPSDIR} make -j4 make install @@ -205,7 +205,7 @@ cd ${BUILDDIR} #tar -xf tcl*.tar.gz #rm *.tar.gz #cd tcl*/unix -#./configure --prefix=${DEPSDIR} +#./configure --host=${ARCH}-linux --prefix=${DEPSDIR} #make -j4 #make install @@ -220,7 +220,7 @@ cd ${BUILDDIR} #tar -xf tk*.tar.gz #rm *.tar.gz #cd tk*/unix -#CFLAGS="-I${DEPSDIR}/include" ./configure --prefix=${DEPSDIR} +#CFLAGS="-I${DEPSDIR}/include" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} #make -j4 #make install From ff36ae090ed729ed5d48b8edb051a2d66af1ec0a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:33:15 -0500 Subject: [PATCH 023/161] add qemu --- .github/workflows/build_python.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index b55c671c..0d2e86b4 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -30,6 +30,10 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + if: ${{ matrix.arch != 'x86_64' }} + - name: Build run: | sudo ./scripts/build_linux_zig.sh ${{ matrix.arch }} ${{ inputs.python_version }} From 5de5563a6b7968d0f67cb4e35e6fc4c3109bae81 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 15:54:43 -0500 Subject: [PATCH 024/161] Disable ncurses stripping --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index f791234c..bae615b5 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -124,7 +124,7 @@ wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* -./configure --host=${ARCH}-linux --with-normal --enable-overwrite --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install From 53bcc6e7e851b5e22134859488e577687eea34a4 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 16:16:25 -0500 Subject: [PATCH 025/161] remove qemu, add cmake system processor flag --- .github/workflows/build_python.yml | 4 ---- scripts/build_linux_zig.sh | 7 ++++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 0d2e86b4..b55c671c 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -30,10 +30,6 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - if: ${{ matrix.arch != 'x86_64' }} - - name: Build run: | sudo ./scripts/build_linux_zig.sh ${{ matrix.arch }} ${{ inputs.python_version }} diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index f791234c..6591a854 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -141,7 +141,7 @@ rm *.tar.gz cd commontk-bzip2* mkdir build cd build -cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. +cmake -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. make -j4 make install @@ -158,7 +158,7 @@ rm *.tar.gz cd xz* mkdir build cd build -cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. +cmake -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. make -j4 make install @@ -175,7 +175,7 @@ rm *.tar.gz cd libuuid* mkdir build cd build -cmake -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. +cmake -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. make -j4 make install @@ -239,6 +239,7 @@ mkdir python-build mkdir python-install cd python-build CFLAGS="-I${DEPSDIR}/include" cmake \ + -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_C_STANDARD=99 \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ -DCMAKE_BUILD_TYPE:STRING=Release \ From b76f63de327391ce7d4ae4873c815dd7078ac1c0 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 16:43:56 -0500 Subject: [PATCH 026/161] tar build folder on exit --- scripts/build_linux_zig.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 8b5c7bae..eaeba19c 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -10,6 +10,8 @@ WORKDIR=$(pwd) BUILDDIR=${WORKDIR}/build DEPSDIR=${WORKDIR}/deps +trap "cd ${BUILDDIR}/python-build && tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz ." EXIT + ######################## # Install dependencies # ######################## @@ -275,8 +277,6 @@ CFLAGS="-I${DEPSDIR}/include" cmake \ make -j4 make install -tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz . - echo "::endgroup::" ############################################# # Check executable dependencies (pre-patch) # From acfc915e97a281e26ba2a1f49db6bdd7cedc9f9d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 20:34:23 -0500 Subject: [PATCH 027/161] add qemu interpreter for platform detection program --- scripts/build_linux_zig.sh | 13 +++++++++++++ scripts/qemu_interpreter | 10 ++++++++++ 2 files changed, 23 insertions(+) create mode 100755 scripts/qemu_interpreter diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index eaeba19c..1dc991d5 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -20,6 +20,18 @@ echo "::group::Install dependencies" export DEBIAN_FRONTEND=noninteractive apt update apt -y install wget build-essential pkg-config cmake autoconf git python3 meson clang patchelf +case "$ARCH" in + x86_64) + apt -y install libc6-amd64-cross + ;; + aarch64) + apt -y install libc6-arm64-cross + ;; + armv7l) + apt -y install libc6-armhf-cross + ;; +esac + cd / wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz @@ -242,6 +254,7 @@ mkdir python-install cd python-build CFLAGS="-I${DEPSDIR}/include" cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ + -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_interpreter \ -DCMAKE_C_STANDARD=99 \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ -DCMAKE_BUILD_TYPE:STRING=Release \ diff --git a/scripts/qemu_interpreter b/scripts/qemu_interpreter new file mode 100755 index 00000000..8a21e149 --- /dev/null +++ b/scripts/qemu_interpreter @@ -0,0 +1,10 @@ +#!/bin/bash +# this interpreter shim is used by cmake to run platform +# detection programs + +# we need to ensure that the binary uses the correct libc +# so first patch the program +patchelf --set-rpath /usr/${ARCH}-linux-gnu/lib/ $1 + +# run the interpreter +qemu-${ARCH}-static $1 \ No newline at end of file From a8b7ebf185e7063fdf7c157397116f0abf3a0802 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 20:37:33 -0500 Subject: [PATCH 028/161] install qemu-user-static --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 1dc991d5..cc515f8b 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -19,7 +19,7 @@ echo "::group::Install dependencies" export DEBIAN_FRONTEND=noninteractive apt update -apt -y install wget build-essential pkg-config cmake autoconf git python3 meson clang patchelf +apt -y install wget build-essential pkg-config cmake autoconf git python3 meson clang patchelf qemu-user-static case "$ARCH" in x86_64) apt -y install libc6-amd64-cross From 71de36bd17f849fce4d35da659ba624463441692 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 20:59:34 -0500 Subject: [PATCH 029/161] use separate interpreter scripts --- scripts/build_linux_zig.sh | 2 +- scripts/{qemu_interpreter => qemu_aarch64_interpreter} | 4 ++-- scripts/qemu_x86_64_interpreter | 10 ++++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) rename scripts/{qemu_interpreter => qemu_aarch64_interpreter} (73%) create mode 100755 scripts/qemu_x86_64_interpreter diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index cc515f8b..8fe9eb02 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -254,7 +254,7 @@ mkdir python-install cd python-build CFLAGS="-I${DEPSDIR}/include" cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ - -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_interpreter \ + -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_C_STANDARD=99 \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ -DCMAKE_BUILD_TYPE:STRING=Release \ diff --git a/scripts/qemu_interpreter b/scripts/qemu_aarch64_interpreter similarity index 73% rename from scripts/qemu_interpreter rename to scripts/qemu_aarch64_interpreter index 8a21e149..2950e00f 100755 --- a/scripts/qemu_interpreter +++ b/scripts/qemu_aarch64_interpreter @@ -4,7 +4,7 @@ # we need to ensure that the binary uses the correct libc # so first patch the program -patchelf --set-rpath /usr/${ARCH}-linux-gnu/lib/ $1 +patchelf --set-rpath /usr/aarch64-linux-gnu/lib/ $1 # run the interpreter -qemu-${ARCH}-static $1 \ No newline at end of file +qemu-aarch64-static $1 \ No newline at end of file diff --git a/scripts/qemu_x86_64_interpreter b/scripts/qemu_x86_64_interpreter new file mode 100755 index 00000000..a2dbb739 --- /dev/null +++ b/scripts/qemu_x86_64_interpreter @@ -0,0 +1,10 @@ +#!/bin/bash +# this interpreter shim is used by cmake to run platform +# detection programs + +# we need to ensure that the binary uses the correct libc +# so first patch the program +patchelf --set-rpath /usr/x86_64-linux-gnu/lib/ $1 + +# run the interpreter +qemu-x86_64-static $1 \ No newline at end of file From a543a8eccbe2f518d0ab054083a0246ceee1d2ff Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 22:47:05 -0500 Subject: [PATCH 030/161] symlink ld, pass all args to qemu --- scripts/build_linux_zig.sh | 2 +- scripts/qemu_aarch64_interpreter | 2 +- scripts/qemu_x86_64_interpreter | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 8fe9eb02..c1a2e1cc 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -31,7 +31,7 @@ case "$ARCH" in apt -y install libc6-armhf-cross ;; esac - +ln -s /usr/${ARCH}-linux-gnu/lib/ld-linux-${ARCH}.so.1 /lib/ld-linux-${ARCH}.so.1 cd / wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz diff --git a/scripts/qemu_aarch64_interpreter b/scripts/qemu_aarch64_interpreter index 2950e00f..018b4fef 100755 --- a/scripts/qemu_aarch64_interpreter +++ b/scripts/qemu_aarch64_interpreter @@ -7,4 +7,4 @@ patchelf --set-rpath /usr/aarch64-linux-gnu/lib/ $1 # run the interpreter -qemu-aarch64-static $1 \ No newline at end of file +qemu-aarch64-static "$@" \ No newline at end of file diff --git a/scripts/qemu_x86_64_interpreter b/scripts/qemu_x86_64_interpreter index a2dbb739..2955ee1a 100755 --- a/scripts/qemu_x86_64_interpreter +++ b/scripts/qemu_x86_64_interpreter @@ -7,4 +7,4 @@ patchelf --set-rpath /usr/x86_64-linux-gnu/lib/ $1 # run the interpreter -qemu-x86_64-static $1 \ No newline at end of file +qemu-x86_64-static "$@" \ No newline at end of file From be1d6890ffb783edb05baaeb98bf157d744a039f Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 22:58:51 -0500 Subject: [PATCH 031/161] don't clobber existing rpath --- scripts/qemu_aarch64_interpreter | 2 +- scripts/qemu_x86_64_interpreter | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/qemu_aarch64_interpreter b/scripts/qemu_aarch64_interpreter index 018b4fef..f37f413d 100755 --- a/scripts/qemu_aarch64_interpreter +++ b/scripts/qemu_aarch64_interpreter @@ -4,7 +4,7 @@ # we need to ensure that the binary uses the correct libc # so first patch the program -patchelf --set-rpath /usr/aarch64-linux-gnu/lib/ $1 +patchelf --add-rpath /usr/aarch64-linux-gnu/lib/ $1 # run the interpreter qemu-aarch64-static "$@" \ No newline at end of file diff --git a/scripts/qemu_x86_64_interpreter b/scripts/qemu_x86_64_interpreter index 2955ee1a..c1e54a20 100755 --- a/scripts/qemu_x86_64_interpreter +++ b/scripts/qemu_x86_64_interpreter @@ -4,7 +4,7 @@ # we need to ensure that the binary uses the correct libc # so first patch the program -patchelf --set-rpath /usr/x86_64-linux-gnu/lib/ $1 +patchelf --add-rpath /usr/x86_64-linux-gnu/lib/ $1 # run the interpreter qemu-x86_64-static "$@" \ No newline at end of file From 3bc974be67b7b7dcd36c52f6c200070babce6df0 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 23:16:45 -0500 Subject: [PATCH 032/161] try adding libm --- scripts/qemu_aarch64_interpreter | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/qemu_aarch64_interpreter b/scripts/qemu_aarch64_interpreter index f37f413d..c15b01d5 100755 --- a/scripts/qemu_aarch64_interpreter +++ b/scripts/qemu_aarch64_interpreter @@ -5,6 +5,7 @@ # we need to ensure that the binary uses the correct libc # so first patch the program patchelf --add-rpath /usr/aarch64-linux-gnu/lib/ $1 +patchelf --add-needed libm.so.6 $1 # run the interpreter qemu-aarch64-static "$@" \ No newline at end of file From a6270f4e119cdad524c22a3d625119faeb2406c4 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 23:27:09 -0500 Subject: [PATCH 033/161] try adding libpthread --- scripts/qemu_aarch64_interpreter | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/qemu_aarch64_interpreter b/scripts/qemu_aarch64_interpreter index c15b01d5..ba1909a6 100755 --- a/scripts/qemu_aarch64_interpreter +++ b/scripts/qemu_aarch64_interpreter @@ -6,6 +6,7 @@ # so first patch the program patchelf --add-rpath /usr/aarch64-linux-gnu/lib/ $1 patchelf --add-needed libm.so.6 $1 +patchelf --add-needed libpthread.so.0 $1 # run the interpreter qemu-aarch64-static "$@" \ No newline at end of file From a7039ba6e78e674fc405f23f177ac2a17734636a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 23:36:08 -0500 Subject: [PATCH 034/161] add more missing libs --- scripts/qemu_aarch64_interpreter | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/qemu_aarch64_interpreter b/scripts/qemu_aarch64_interpreter index ba1909a6..6363988b 100755 --- a/scripts/qemu_aarch64_interpreter +++ b/scripts/qemu_aarch64_interpreter @@ -7,6 +7,8 @@ patchelf --add-rpath /usr/aarch64-linux-gnu/lib/ $1 patchelf --add-needed libm.so.6 $1 patchelf --add-needed libpthread.so.0 $1 +patchelf --add-needed libdl.so.2 $1 +patchelf --add-needed libutil.so.1 $1 # run the interpreter qemu-aarch64-static "$@" \ No newline at end of file From 3134ddb6c72a8e818ea093a4de1e80cff4022a1e Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 26 Jan 2024 23:40:49 -0500 Subject: [PATCH 035/161] initial refactoring for armv7 --- scripts/build_linux_zig.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index c1a2e1cc..bccf66c4 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -23,15 +23,17 @@ apt -y install wget build-essential pkg-config cmake autoconf git python3 meson case "$ARCH" in x86_64) apt -y install libc6-amd64-cross + ln -s /usr/x86_64-linux-gnu/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 ;; aarch64) apt -y install libc6-arm64-cross + ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/ld-linux-aarch64.so.1 ;; - armv7l) + arm) apt -y install libc6-armhf-cross + ln -s /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3 ;; esac -ln -s /usr/${ARCH}-linux-gnu/lib/ld-linux-${ARCH}.so.1 /lib/ld-linux-${ARCH}.so.1 cd / wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz From 122b6df3165dd28880c3b8a06d3b684869580940 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 00:26:37 -0500 Subject: [PATCH 036/161] Tweak elf dep checking and running compiled result --- scripts/build_linux_zig.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index bccf66c4..9f9f132f 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -301,10 +301,10 @@ cd ${BUILDDIR} cd python-install echo "python dependencies" -ldd -v -r ./bin/python +readelf -d ./bin/python echo echo "libpython dependencies" -ldd -v -r ./lib/libpython${PYTHON_VER}.so +readelf -d ./lib/libpython${PYTHON_VER}.so echo "::endgroup::" ################ @@ -325,8 +325,8 @@ echo "::group::Check executable dependencies (post-patch)" cd ${BUILDDIR} cd python-install -# we don't make ldd errors fatal here since sometimes ldd can -# crash but the patched binaries still work +# we don't make ldd errors fatal here since ldd doesn't work +# when cross compiling echo "python dependencies" ldd -v -r ./bin/python || true echo @@ -341,7 +341,7 @@ echo "::group::Test python" cd ${BUILDDIR} cd python-install -./bin/python --version +qemu-${ARCH}-static ./bin/python --version echo "::endgroup::" ############### @@ -351,7 +351,7 @@ echo "::group::Preload pip" cd ${BUILDDIR} cd python-install -./bin/python -m ensurepip +qemu-${ARCH}-static ./bin/python -m ensurepip echo "::endgroup::" ################### From 0cb9ead609bcf5d47607322bc40b9e4e0df869e6 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 07:36:59 -0500 Subject: [PATCH 037/161] initial pass at arm cross compile --- .github/workflows/build_python.yml | 2 +- scripts/qemu_arm_interpreter | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100755 scripts/qemu_arm_interpreter diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index b55c671c..cf13c723 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - arch: [ x86_64, aarch64 ] + arch: [ x86_64, aarch64, arm ] steps: - name: Checkout diff --git a/scripts/qemu_arm_interpreter b/scripts/qemu_arm_interpreter new file mode 100755 index 00000000..dfffe8df --- /dev/null +++ b/scripts/qemu_arm_interpreter @@ -0,0 +1,14 @@ +#!/bin/bash +# this interpreter shim is used by cmake to run platform +# detection programs + +# we need to ensure that the binary uses the correct libc +# so first patch the program +patchelf --add-rpath /usr/arm-linux-gnueabihf/lib/ $1 +patchelf --add-needed libm.so.6 $1 +patchelf --add-needed libpthread.so.0 $1 +patchelf --add-needed libdl.so.2 $1 +patchelf --add-needed libutil.so.1 $1 + +# run the interpreter +qemu-arm-static "$@" From 78f224bc8fd8b8cfa7e8dfddc3aeafa371297d18 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 08:01:57 -0500 Subject: [PATCH 038/161] set gnueabihf --- scripts/build_linux_zig.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 9f9f132f..ac51a0b9 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -24,14 +24,17 @@ case "$ARCH" in x86_64) apt -y install libc6-amd64-cross ln -s /usr/x86_64-linux-gnu/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 + export TARGET=${ARCH}-linux-gnu.2.17 ;; aarch64) apt -y install libc6-arm64-cross ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/ld-linux-aarch64.so.1 + export TARGET=${ARCH}-linux-gnu.2.17 ;; arm) apt -y install libc6-armhf-cross ln -s /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3 + export TARGET=${ARCH}-linux-gnueabihf.2.17 ;; esac @@ -51,7 +54,6 @@ export CC=zig_cc export CXX=zig_cxx export CHOST=${ARCH} -export TARGET=${ARCH}-linux-gnu.2.17 export ZIG_TARGET=${TARGET} echo "::endgroup::" @@ -366,4 +368,4 @@ mv python-install python-${PYTHON_FULL_VER}-linux-${ARCH} tar -czf ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz python-${PYTHON_FULL_VER}-linux-${ARCH} zip ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.zip $(tar tf ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz) -echo "::endgroup::" \ No newline at end of file +echo "::endgroup::" From 63f533da80b2a2a9e37a615ab7af09c1610bf8a2 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 08:10:01 -0500 Subject: [PATCH 039/161] arm fixes --- scripts/build_linux_zig.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index ac51a0b9..4eaaeec9 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -24,17 +24,14 @@ case "$ARCH" in x86_64) apt -y install libc6-amd64-cross ln -s /usr/x86_64-linux-gnu/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 - export TARGET=${ARCH}-linux-gnu.2.17 ;; aarch64) apt -y install libc6-arm64-cross ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/ld-linux-aarch64.so.1 - export TARGET=${ARCH}-linux-gnu.2.17 ;; arm) apt -y install libc6-armhf-cross ln -s /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3 - export TARGET=${ARCH}-linux-gnueabihf.2.17 ;; esac @@ -54,7 +51,14 @@ export CC=zig_cc export CXX=zig_cxx export CHOST=${ARCH} -export ZIG_TARGET=${TARGET} +case "$ARCH" in + arm) + export ZIG_TARGET=${ARCH}-linux-gnueabihf.2.17 + ;; + *) + export ZIG_TARGET=${ARCH}-linux-gnu.2.17 + ;; +esac echo "::endgroup::" ######## @@ -82,7 +86,14 @@ wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar. tar -xf OpenSSL*.tar.gz rm *.tar.gz cd openssl-OpenSSL* -./Configure linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} +case "$ARCH" in + arm) + ./Configure linux-generic32 no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} + ;; + *) + ./Configure linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} + ;; +esac make -j4 make install_sw From dfce226068ae74350235d2f32834f00577ed161a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 09:38:02 -0500 Subject: [PATCH 040/161] gnueabihf CHOST --- scripts/build_linux_zig.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 4eaaeec9..8c5ea715 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -49,14 +49,15 @@ mkdir ${DEPSDIR} export AR=zig_ar export CC=zig_cc export CXX=zig_cxx -export CHOST=${ARCH} case "$ARCH" in arm) export ZIG_TARGET=${ARCH}-linux-gnueabihf.2.17 + export CHOST=arm-linux-gnueabihf ;; *) export ZIG_TARGET=${ARCH}-linux-gnu.2.17 + export CHOST=${ARCH} ;; esac @@ -108,7 +109,7 @@ wget -q https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.t tar -xf libffi*.tar.gz rm *.tar.gz cd libffi* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --host=${CHOST} --prefix=${DEPSDIR} make -j4 make install @@ -123,7 +124,7 @@ wget -q https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz tar -xf sqlite*.tar.gz rm *.tar.gz cd sqlite* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --host=${CHOST} --prefix=${DEPSDIR} make -j4 make install @@ -138,7 +139,7 @@ wget -q https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz tar -xf readline*.tar.gz rm *.tar.gz cd readline* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --host=${CHOST} --prefix=${DEPSDIR} make -j4 make install @@ -153,7 +154,7 @@ wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* -./configure --host=${ARCH}-linux --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +./configure --host=${CHOST} --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install @@ -219,7 +220,7 @@ wget -q https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz tar -xf gdbm*.tar.gz rm *.tar.gz cd gdbm* -./configure --host=${ARCH}-linux --enable-libgdbm-compat --prefix=${DEPSDIR} +./configure --host=${CHOST} --enable-libgdbm-compat --prefix=${DEPSDIR} make -j4 make install @@ -234,7 +235,7 @@ cd ${BUILDDIR} #tar -xf tcl*.tar.gz #rm *.tar.gz #cd tcl*/unix -#./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +#./configure --host=${CHOST} --prefix=${DEPSDIR} #make -j4 #make install @@ -249,7 +250,7 @@ cd ${BUILDDIR} #tar -xf tk*.tar.gz #rm *.tar.gz #cd tk*/unix -#CFLAGS="-I${DEPSDIR}/include" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +#CFLAGS="-I${DEPSDIR}/include" ./configure --host=${CHOST} --prefix=${DEPSDIR} #make -j4 #make install From 1b6702a9d919a5b23717ee16075e9799558e0197 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 11:38:52 -0500 Subject: [PATCH 041/161] arm flags --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 8c5ea715..27579f06 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -52,7 +52,7 @@ export CXX=zig_cxx case "$ARCH" in arm) - export ZIG_TARGET=${ARCH}-linux-gnueabihf.2.17 + export ZIG_TARGET="${ARCH}-linux-gnueabihf.2.17 -march=armv6 -mfpu=vfp -mfloat-abi=hard" export CHOST=arm-linux-gnueabihf ;; *) From 9beed29c39ca9511297767deacb4b93784725813 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 11:48:44 -0500 Subject: [PATCH 042/161] tweak arm flags --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 27579f06..ed9848e8 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -52,7 +52,7 @@ export CXX=zig_cxx case "$ARCH" in arm) - export ZIG_TARGET="${ARCH}-linux-gnueabihf.2.17 -march=armv6 -mfpu=vfp -mfloat-abi=hard" + export ZIG_TARGET="${ARCH}-linux-gnueabihf.2.17 -mcpu=generic+v7a+vfp3-d32+thumb2-neon" export CHOST=arm-linux-gnueabihf ;; *) From 9659aa62862c3b6a91073433bb5df8c046daaffc Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 13:05:05 -0500 Subject: [PATCH 043/161] Revert "tweak arm flags" This reverts commit 9beed29c39ca9511297767deacb4b93784725813. --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index ed9848e8..27579f06 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -52,7 +52,7 @@ export CXX=zig_cxx case "$ARCH" in arm) - export ZIG_TARGET="${ARCH}-linux-gnueabihf.2.17 -mcpu=generic+v7a+vfp3-d32+thumb2-neon" + export ZIG_TARGET="${ARCH}-linux-gnueabihf.2.17 -march=armv6 -mfpu=vfp -mfloat-abi=hard" export CHOST=arm-linux-gnueabihf ;; *) From 3fb052064a151da0a6e3082046c97bb1cd109c5f Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 13:05:14 -0500 Subject: [PATCH 044/161] Revert "arm flags" This reverts commit 1b6702a9d919a5b23717ee16075e9799558e0197. --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 27579f06..8c5ea715 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -52,7 +52,7 @@ export CXX=zig_cxx case "$ARCH" in arm) - export ZIG_TARGET="${ARCH}-linux-gnueabihf.2.17 -march=armv6 -mfpu=vfp -mfloat-abi=hard" + export ZIG_TARGET=${ARCH}-linux-gnueabihf.2.17 export CHOST=arm-linux-gnueabihf ;; *) From bfb670cc2e089c58214740d5b70b86c07f0b291f Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 13:05:16 -0500 Subject: [PATCH 045/161] Revert "gnueabihf CHOST" This reverts commit dfce226068ae74350235d2f32834f00577ed161a. --- scripts/build_linux_zig.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 8c5ea715..4eaaeec9 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -49,15 +49,14 @@ mkdir ${DEPSDIR} export AR=zig_ar export CC=zig_cc export CXX=zig_cxx +export CHOST=${ARCH} case "$ARCH" in arm) export ZIG_TARGET=${ARCH}-linux-gnueabihf.2.17 - export CHOST=arm-linux-gnueabihf ;; *) export ZIG_TARGET=${ARCH}-linux-gnu.2.17 - export CHOST=${ARCH} ;; esac @@ -109,7 +108,7 @@ wget -q https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.t tar -xf libffi*.tar.gz rm *.tar.gz cd libffi* -./configure --host=${CHOST} --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -124,7 +123,7 @@ wget -q https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz tar -xf sqlite*.tar.gz rm *.tar.gz cd sqlite* -./configure --host=${CHOST} --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -139,7 +138,7 @@ wget -q https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz tar -xf readline*.tar.gz rm *.tar.gz cd readline* -./configure --host=${CHOST} --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -154,7 +153,7 @@ wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* -./configure --host=${CHOST} --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install @@ -220,7 +219,7 @@ wget -q https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz tar -xf gdbm*.tar.gz rm *.tar.gz cd gdbm* -./configure --host=${CHOST} --enable-libgdbm-compat --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --enable-libgdbm-compat --prefix=${DEPSDIR} make -j4 make install @@ -235,7 +234,7 @@ cd ${BUILDDIR} #tar -xf tcl*.tar.gz #rm *.tar.gz #cd tcl*/unix -#./configure --host=${CHOST} --prefix=${DEPSDIR} +#./configure --host=${ARCH}-linux --prefix=${DEPSDIR} #make -j4 #make install @@ -250,7 +249,7 @@ cd ${BUILDDIR} #tar -xf tk*.tar.gz #rm *.tar.gz #cd tk*/unix -#CFLAGS="-I${DEPSDIR}/include" ./configure --host=${CHOST} --prefix=${DEPSDIR} +#CFLAGS="-I${DEPSDIR}/include" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} #make -j4 #make install From dcd15c6faebf0d13c68b1acabba886b61f78355d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 13:05:18 -0500 Subject: [PATCH 046/161] Revert "arm fixes" This reverts commit 63f533da80b2a2a9e37a615ab7af09c1610bf8a2. --- scripts/build_linux_zig.sh | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 4eaaeec9..ac51a0b9 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -24,14 +24,17 @@ case "$ARCH" in x86_64) apt -y install libc6-amd64-cross ln -s /usr/x86_64-linux-gnu/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 + export TARGET=${ARCH}-linux-gnu.2.17 ;; aarch64) apt -y install libc6-arm64-cross ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/ld-linux-aarch64.so.1 + export TARGET=${ARCH}-linux-gnu.2.17 ;; arm) apt -y install libc6-armhf-cross ln -s /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3 + export TARGET=${ARCH}-linux-gnueabihf.2.17 ;; esac @@ -51,14 +54,7 @@ export CC=zig_cc export CXX=zig_cxx export CHOST=${ARCH} -case "$ARCH" in - arm) - export ZIG_TARGET=${ARCH}-linux-gnueabihf.2.17 - ;; - *) - export ZIG_TARGET=${ARCH}-linux-gnu.2.17 - ;; -esac +export ZIG_TARGET=${TARGET} echo "::endgroup::" ######## @@ -86,14 +82,7 @@ wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar. tar -xf OpenSSL*.tar.gz rm *.tar.gz cd openssl-OpenSSL* -case "$ARCH" in - arm) - ./Configure linux-generic32 no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} - ;; - *) - ./Configure linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} - ;; -esac +./Configure linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} make -j4 make install_sw From d6496ffebdbaf3146a26be774decdae55f9fc45f Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 13:05:20 -0500 Subject: [PATCH 047/161] Revert "set gnueabihf" This reverts commit 78f224bc8fd8b8cfa7e8dfddc3aeafa371297d18. --- scripts/build_linux_zig.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index ac51a0b9..9f9f132f 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -24,17 +24,14 @@ case "$ARCH" in x86_64) apt -y install libc6-amd64-cross ln -s /usr/x86_64-linux-gnu/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 - export TARGET=${ARCH}-linux-gnu.2.17 ;; aarch64) apt -y install libc6-arm64-cross ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/ld-linux-aarch64.so.1 - export TARGET=${ARCH}-linux-gnu.2.17 ;; arm) apt -y install libc6-armhf-cross ln -s /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3 - export TARGET=${ARCH}-linux-gnueabihf.2.17 ;; esac @@ -54,6 +51,7 @@ export CC=zig_cc export CXX=zig_cxx export CHOST=${ARCH} +export TARGET=${ARCH}-linux-gnu.2.17 export ZIG_TARGET=${TARGET} echo "::endgroup::" @@ -368,4 +366,4 @@ mv python-install python-${PYTHON_FULL_VER}-linux-${ARCH} tar -czf ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz python-${PYTHON_FULL_VER}-linux-${ARCH} zip ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.zip $(tar tf ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz) -echo "::endgroup::" +echo "::endgroup::" \ No newline at end of file From ca8d4aad97eafae1658b3600e8e598e40a25b72e Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 13:05:46 -0500 Subject: [PATCH 048/161] Revert "initial pass at arm cross compile" This reverts commit 0cb9ead609bcf5d47607322bc40b9e4e0df869e6. --- .github/workflows/build_python.yml | 2 +- scripts/qemu_arm_interpreter | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) delete mode 100755 scripts/qemu_arm_interpreter diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index cf13c723..b55c671c 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - arch: [ x86_64, aarch64, arm ] + arch: [ x86_64, aarch64 ] steps: - name: Checkout diff --git a/scripts/qemu_arm_interpreter b/scripts/qemu_arm_interpreter deleted file mode 100755 index dfffe8df..00000000 --- a/scripts/qemu_arm_interpreter +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# this interpreter shim is used by cmake to run platform -# detection programs - -# we need to ensure that the binary uses the correct libc -# so first patch the program -patchelf --add-rpath /usr/arm-linux-gnueabihf/lib/ $1 -patchelf --add-needed libm.so.6 $1 -patchelf --add-needed libpthread.so.0 $1 -patchelf --add-needed libdl.so.2 $1 -patchelf --add-needed libutil.so.1 $1 - -# run the interpreter -qemu-arm-static "$@" From 1b59f06e88dd0bc938ef617765510ac0aed70860 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 14:19:45 -0500 Subject: [PATCH 049/161] build armv7l with old method --- .github/workflows/build_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index b55c671c..b99beef1 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -48,7 +48,7 @@ jobs: path: ./*python*.tar.gz build_linux: - if: ${{ false && startsWith(inputs.python_version, '3.8') }} + if: ${{ startsWith(inputs.python_version, '3.8') || matrix.arch == 'armv7l' }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} runs-on: ubuntu-latest strategy: From 16fa71a1cafa1eb2360377b16c58ded3e2714d80 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 14:20:52 -0500 Subject: [PATCH 050/161] Revert "build armv7l with old method" This reverts commit 1b59f06e88dd0bc938ef617765510ac0aed70860. --- .github/workflows/build_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index b99beef1..b55c671c 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -48,7 +48,7 @@ jobs: path: ./*python*.tar.gz build_linux: - if: ${{ startsWith(inputs.python_version, '3.8') || matrix.arch == 'armv7l' }} + if: ${{ false && startsWith(inputs.python_version, '3.8') }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} runs-on: ubuntu-latest strategy: From 17e709cf00527ac734649bf36e3fc314ea5db113 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 14:45:09 -0500 Subject: [PATCH 051/161] refactor workflow --- .github/workflows/build_python.yml | 112 +++---------------- .github/workflows/build_python_linux.yml | 83 ++++++++++++++ .github/workflows/build_python_linux_zig.yml | 37 ++++++ 3 files changed, 138 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/build_python_linux.yml create mode 100644 .github/workflows/build_python_linux_zig.yml diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index b55c671c..73eb124f 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -12,115 +12,40 @@ on: required: true type: string -env: - image_map: '{"x86_64": "debian:buster", "aarch64": "debian:buster", "armv7l": "debian:buster"}' - platform_map: '{"x86_64": "linux/amd64", "aarch64": "linux/arm64/v8", "armv7l": "linux/arm/v7"}' - jobs: build_linux_zig: if: ${{ !startsWith(inputs.python_version, '3.8') }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} - runs-on: ubuntu-latest strategy: fail-fast: false matrix: arch: [ x86_64, aarch64 ] + uses: ./.github/workflows/build_python_linux_zig.yml + with: + python_version: ${{ inputs.python_version }} + arch: ${{ matrix.arch }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Build - run: | - sudo ./scripts/build_linux_zig.sh ${{ matrix.arch }} ${{ inputs.python_version }} - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: python - path: ./python*.zip - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: build-python - path: ./*python*.tar.gz - - build_linux: - if: ${{ false && startsWith(inputs.python_version, '3.8') }} + build_linux_armv7l: + if: ${{ !startsWith(inputs.python_version, '3.8') }} + name: Linux ${{ inputs.python_version }} armv7l + uses: ./.github/workflows/build_python_linux.yml + with: + python_version: ${{ inputs.python_version }} + arch: armv7l + + build_linux_3_8: + if: ${{ startsWith(inputs.python_version, '3.8') }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} - runs-on: ubuntu-latest strategy: fail-fast: false matrix: arch: [ x86_64, aarch64, armv7l ] - - steps: - - name: Parse image - id: parse_image - run: | - IMAGE=$(echo ${{ toJSON(env.image_map) }} | jq -r '.["${{ matrix.arch }}"]') - echo "::set-output name=image::$IMAGE" - - - name: Parse platform - id: parse_platform - run: | - PLATFORM=$(echo ${{ toJSON(env.platform_map) }} | jq -r '.["${{ matrix.arch }}"]') - echo "::set-output name=platform::$PLATFORM" - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - if: ${{ matrix.arch != 'x86_64' }} - - - name: Build - uses: addnab/docker-run-action@v3 - with: - image: ${{ steps.parse_image.outputs.image }} - options: -v ${{ github.workspace }}:/work --workdir /tmp --platform ${{ steps.parse_platform.outputs.platform }} - run: | - set -e - /work/scripts/build_linux.sh ${{ matrix.arch }} ${{ inputs.python_version }} - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: python - path: ./python*.zip - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: build-python - path: ./*python*.tar.gz - - - name: Test python in clean environment - uses: addnab/docker-run-action@v3 - with: - image: ${{ steps.parse_image.outputs.image }} - options: -v ${{ github.workspace }}:/work --workdir /tmp --platform ${{ steps.parse_platform.outputs.platform }} - run: | - set -e - apt update - apt install -y unzip - - cp /work/python*.zip . - unzip ./python*.zip - - cd python-${{ inputs.python_version }}-linux-${{ matrix.arch }} - chmod +x ./bin/python - ldd -v -r ./bin/python || true - ./bin/python --version - - # make tests nonfatal for now - # ./bin/python -m test || true + uses: ./.github/workflows/build_python_linux.yml + with: + python_version: ${{ inputs.python_version }} + arch: ${{ matrix.arch }} build_windows: - if: false name: Windows ${{ inputs.python_version }} x86_64 (build) runs-on: windows-latest @@ -180,7 +105,6 @@ jobs: # ./bin/python -m test || true build_macos: - if: false name: MacOS ${{ inputs.python_version }} universal2 (build) runs-on: macos-latest diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml new file mode 100644 index 00000000..9da93e38 --- /dev/null +++ b/.github/workflows/build_python_linux.yml @@ -0,0 +1,83 @@ +name: Build Python + +on: + workflow_call: + inputs: + python_version: + required: true + type: string + arch: + required: true + type: string + +env: + image_map: '{"x86_64": "debian:buster", "aarch64": "debian:buster", "armv7l": "debian:buster"}' + platform_map: '{"x86_64": "linux/amd64", "aarch64": "linux/arm64/v8", "armv7l": "linux/arm/v7"}' + +jobs: + build_linux: + name: Linux ${{ inputs.python_version }} ${{ inputs.arch }} + runs-on: ubuntu-latest + + steps: + - name: Parse image + id: parse_image + run: | + IMAGE=$(echo ${{ toJSON(env.image_map) }} | jq -r '.["${{ inputs.arch }}"]') + echo "::set-output name=image::$IMAGE" + + - name: Parse platform + id: parse_platform + run: | + PLATFORM=$(echo ${{ toJSON(env.platform_map) }} | jq -r '.["${{ inputs.arch }}"]') + echo "::set-output name=platform::$PLATFORM" + + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + if: ${{ inputs.arch != 'x86_64' }} + + - name: Build + uses: addnab/docker-run-action@v3 + with: + image: ${{ steps.parse_image.outputs.image }} + options: -v ${{ github.workspace }}:/work --workdir /tmp --platform ${{ steps.parse_platform.outputs.platform }} + run: | + set -e + /work/scripts/build_linux.sh ${{ inputs.arch }} ${{ inputs.python_version }} + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: python + path: ./python*.zip + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: build-python + path: ./*python*.tar.gz + + - name: Test python in clean environment + uses: addnab/docker-run-action@v3 + with: + image: ${{ steps.parse_image.outputs.image }} + options: -v ${{ github.workspace }}:/work --workdir /tmp --platform ${{ steps.parse_platform.outputs.platform }} + run: | + set -e + apt update + apt install -y unzip + + cp /work/python*.zip . + unzip ./python*.zip + + cd python-${{ inputs.python_version }}-linux-${{ inputs.arch }} + chmod +x ./bin/python + ldd -v -r ./bin/python || true + ./bin/python --version + + # make tests nonfatal for now + # ./bin/python -m test || true \ No newline at end of file diff --git a/.github/workflows/build_python_linux_zig.yml b/.github/workflows/build_python_linux_zig.yml new file mode 100644 index 00000000..531a2b19 --- /dev/null +++ b/.github/workflows/build_python_linux_zig.yml @@ -0,0 +1,37 @@ +name: Build Python + +on: + workflow_call: + inputs: + python_version: + required: true + type: string + arch: + required: true + type: string + +jobs: + build_linux_zig: + name: Linux ${{ inputs.python_version }} ${{ inputs.arch }} + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build + run: | + sudo ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: python + path: ./python*.zip + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: build-python + path: ./*python*.tar.gz \ No newline at end of file From dbc0fa531e3c2b005a61c04e3964bbcbfcddce22 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 14:47:47 -0500 Subject: [PATCH 052/161] make workflow names more descriptive --- .github/workflows/build_python.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 73eb124f..e0a092be 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -15,7 +15,7 @@ on: jobs: build_linux_zig: if: ${{ !startsWith(inputs.python_version, '3.8') }} - name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} + name: Linux (Zig) ${{ inputs.python_version }} ${{ matrix.arch }} strategy: fail-fast: false matrix: @@ -27,7 +27,7 @@ jobs: build_linux_armv7l: if: ${{ !startsWith(inputs.python_version, '3.8') }} - name: Linux ${{ inputs.python_version }} armv7l + name: Linux (Docker) ${{ inputs.python_version }} armv7l uses: ./.github/workflows/build_python_linux.yml with: python_version: ${{ inputs.python_version }} @@ -35,7 +35,7 @@ jobs: build_linux_3_8: if: ${{ startsWith(inputs.python_version, '3.8') }} - name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} + name: Linux (Docker) ${{ inputs.python_version }} ${{ matrix.arch }} strategy: fail-fast: false matrix: From 9eaca7fea8c6cf47bffb45491f1ce7b7a2cc34c2 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 14:53:54 -0500 Subject: [PATCH 053/161] refactor and consolidate --- .github/workflows/build_python.yml | 24 ++----------- .github/workflows/build_python_linux.yml | 29 ++++++++++++++- .github/workflows/build_python_linux_zig.yml | 37 -------------------- 3 files changed, 30 insertions(+), 60 deletions(-) delete mode 100644 .github/workflows/build_python_linux_zig.yml diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index e0a092be..d247621d 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -13,29 +13,9 @@ on: type: string jobs: - build_linux_zig: + build_linux: if: ${{ !startsWith(inputs.python_version, '3.8') }} - name: Linux (Zig) ${{ inputs.python_version }} ${{ matrix.arch }} - strategy: - fail-fast: false - matrix: - arch: [ x86_64, aarch64 ] - uses: ./.github/workflows/build_python_linux_zig.yml - with: - python_version: ${{ inputs.python_version }} - arch: ${{ matrix.arch }} - - build_linux_armv7l: - if: ${{ !startsWith(inputs.python_version, '3.8') }} - name: Linux (Docker) ${{ inputs.python_version }} armv7l - uses: ./.github/workflows/build_python_linux.yml - with: - python_version: ${{ inputs.python_version }} - arch: armv7l - - build_linux_3_8: - if: ${{ startsWith(inputs.python_version, '3.8') }} - name: Linux (Docker) ${{ inputs.python_version }} ${{ matrix.arch }} + name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index 9da93e38..600bf4c1 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -15,8 +15,35 @@ env: platform_map: '{"x86_64": "linux/amd64", "aarch64": "linux/arm64/v8", "armv7l": "linux/arm/v7"}' jobs: + build_linux_zig: + if: ${{ !startsWith(inputs.python_version, '3.8') && inputs.arch != 'armv7l' }} + name: Linux (Zig) ${{ inputs.python_version }} ${{ inputs.arch }} + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build + run: | + sudo ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: python + path: ./python*.zip + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: build-python + path: ./*python*.tar.gz + build_linux: - name: Linux ${{ inputs.python_version }} ${{ inputs.arch }} + if: ${{ startsWith(inputs.python_version, '3.8') || inputs.arch == 'armv7l' }} + name: Linux (Docker) ${{ inputs.python_version }} ${{ inputs.arch }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/build_python_linux_zig.yml b/.github/workflows/build_python_linux_zig.yml deleted file mode 100644 index 531a2b19..00000000 --- a/.github/workflows/build_python_linux_zig.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build Python - -on: - workflow_call: - inputs: - python_version: - required: true - type: string - arch: - required: true - type: string - -jobs: - build_linux_zig: - name: Linux ${{ inputs.python_version }} ${{ inputs.arch }} - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Build - run: | - sudo ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: python - path: ./python*.zip - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: build-python - path: ./*python*.tar.gz \ No newline at end of file From 14c91186bfbb68e0c00b3b0503cc36aa6926aefe Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 14:54:23 -0500 Subject: [PATCH 054/161] remmove condition --- .github/workflows/build_python.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index d247621d..d28fa6a9 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -14,7 +14,6 @@ on: jobs: build_linux: - if: ${{ !startsWith(inputs.python_version, '3.8') }} name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} strategy: fail-fast: false From 809f215c5a8d457c34eb63ded874b57d1e47cacc Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 15:14:38 -0500 Subject: [PATCH 055/161] test zig builds --- .github/workflows/build_python_linux.yml | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index 600bf4c1..1c53ec7e 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -21,6 +21,12 @@ jobs: runs-on: ubuntu-latest steps: + - name: Parse platform + id: parse_platform + run: | + PLATFORM=$(echo ${{ toJSON(env.platform_map) }} | jq -r '.["${{ inputs.arch }}"]') + echo "::set-output name=platform::$PLATFORM" + - name: Checkout uses: actions/checkout@v3 @@ -41,6 +47,33 @@ jobs: name: build-python path: ./*python*.tar.gz + - name: Test python in clean environment + uses: addnab/docker-run-action@v3 + with: + image: centos:7 + options: -v ${{ github.workspace }}:/work --workdir /tmp --platform ${{ steps.parse_platform.outputs.platform }} + run: | + set -e + + if [[ "${{ inputs.arch }}" == "armv7l" ]]; then + echo "armhfp" > /etc/yum/vars/basearch + echo "armv7hl" > /etc/yum/vars/arch + echo "armv7hl-redhat-linux-gpu" > /etc/rpm/platform + fi + + yum -y install unzip + + cp /work/python*.zip . + unzip ./python*.zip + + cd python-${{ inputs.python_version }}-linux-${{ inputs.arch }} + chmod +x ./bin/python + ldd -v -r ./bin/python || true + ./bin/python --version + + # make tests nonfatal for now + # ./bin/python -m test || true + build_linux: if: ${{ startsWith(inputs.python_version, '3.8') || inputs.arch == 'armv7l' }} name: Linux (Docker) ${{ inputs.python_version }} ${{ inputs.arch }} From 1648ab6caec55756b4c1011aa931c7be3b900422 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 15:15:59 -0500 Subject: [PATCH 056/161] test ssl on all platforms --- .github/workflows/build_python_linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index 1c53ec7e..c73d5a2b 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -70,6 +70,7 @@ jobs: chmod +x ./bin/python ldd -v -r ./bin/python || true ./bin/python --version + ./bin/python -c "import ssl" # make tests nonfatal for now # ./bin/python -m test || true @@ -138,6 +139,7 @@ jobs: chmod +x ./bin/python ldd -v -r ./bin/python || true ./bin/python --version + ./bin/python -c "import ssl" # make tests nonfatal for now # ./bin/python -m test || true \ No newline at end of file From 538fba614ae0b928b9547c1d69911c0f1fd74747 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 15:58:34 -0500 Subject: [PATCH 057/161] Use zig 0.10.0 --- scripts/build_linux_zig.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 9f9f132f..e7131316 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -36,12 +36,12 @@ case "$ARCH" in esac cd / -wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz +wget -q https://ziglang.org/download/0.10.0/zig-linux-x86_64-0.10.0.tar.xz tar -xf zig*.tar.xz cd ${WORKDIR} -cp -r zigshim/* /zig-linux-x86_64-0.11.0 -export PATH=${PATH}:/zig-linux-x86_64-0.11.0 +cp -r zigshim/* /zig-linux-x86_64-0.10.0 +export PATH=${PATH}:/zig-linux-x86_64-0.10.0 mkdir ${BUILDDIR} mkdir ${DEPSDIR} From 4acf5de9f3e9dd0e6b2cef203cb85989f5ef0939 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 17:47:32 -0500 Subject: [PATCH 058/161] Revert "Use zig 0.10.0" This reverts commit 538fba614ae0b928b9547c1d69911c0f1fd74747. --- scripts/build_linux_zig.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index e7131316..9f9f132f 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -36,12 +36,12 @@ case "$ARCH" in esac cd / -wget -q https://ziglang.org/download/0.10.0/zig-linux-x86_64-0.10.0.tar.xz +wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz tar -xf zig*.tar.xz cd ${WORKDIR} -cp -r zigshim/* /zig-linux-x86_64-0.10.0 -export PATH=${PATH}:/zig-linux-x86_64-0.10.0 +cp -r zigshim/* /zig-linux-x86_64-0.11.0 +export PATH=${PATH}:/zig-linux-x86_64-0.11.0 mkdir ${BUILDDIR} mkdir ${DEPSDIR} From 974b6cd8cdf41e98b97ae0a3684c8fc1fbab38e2 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 19:26:56 -0500 Subject: [PATCH 059/161] add ninja --- scripts/build_linux_zig.sh | 3 ++- zigshim/zig_cc | 7 +------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 9f9f132f..36d5f780 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -19,7 +19,7 @@ echo "::group::Install dependencies" export DEBIAN_FRONTEND=noninteractive apt update -apt -y install wget build-essential pkg-config cmake autoconf git python3 meson clang patchelf qemu-user-static +apt -y install wget build-essential pkg-config cmake autoconf git python3 python3-pip clang patchelf qemu-user-static case "$ARCH" in x86_64) apt -y install libc6-amd64-cross @@ -34,6 +34,7 @@ case "$ARCH" in ln -s /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3 ;; esac +pip install https://github.com/mesonbuild/meson/archive/2baae24.zip ninja cd / wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz diff --git a/zigshim/zig_cc b/zigshim/zig_cc index 97a42a4f..5e5302c2 100755 --- a/zigshim/zig_cc +++ b/zigshim/zig_cc @@ -1,7 +1,2 @@ #!/bin/bash -if [ "$1" == "-Wl,--version" ]; then - # hack for meson to detect ld correctly - clang "$@" -else - zig cc -target $ZIG_TARGET "$@" -fi \ No newline at end of file +zig cc -target $ZIG_TARGET "$@" \ No newline at end of file From b6db3a43d69b7f6e5d812936635ba7d7ff4f5f2c Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 21:52:17 -0500 Subject: [PATCH 060/161] swap zig setup --- .github/workflows/build_python_linux.yml | 5 ++++- scripts/build_linux_zig.sh | 21 ++++++--------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index c73d5a2b..f31a4b87 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -27,6 +27,9 @@ jobs: PLATFORM=$(echo ${{ toJSON(env.platform_map) }} | jq -r '.["${{ inputs.arch }}"]') echo "::set-output name=platform::$PLATFORM" + - name: Set up zig + uses: goto-bus-stop/setup-zig@v2 + - name: Checkout uses: actions/checkout@v3 @@ -142,4 +145,4 @@ jobs: ./bin/python -c "import ssl" # make tests nonfatal for now - # ./bin/python -m test || true \ No newline at end of file + # ./bin/python -m test || true diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 36d5f780..5d97af7d 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -36,24 +36,15 @@ case "$ARCH" in esac pip install https://github.com/mesonbuild/meson/archive/2baae24.zip ninja -cd / -wget -q https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz -tar -xf zig*.tar.xz -cd ${WORKDIR} - -cp -r zigshim/* /zig-linux-x86_64-0.11.0 -export PATH=${PATH}:/zig-linux-x86_64-0.11.0 - mkdir ${BUILDDIR} mkdir ${DEPSDIR} -export AR=zig_ar -export CC=zig_cc -export CXX=zig_cxx -export CHOST=${ARCH} - export TARGET=${ARCH}-linux-gnu.2.17 -export ZIG_TARGET=${TARGET} + +export AR="zig ar" +export CC="zig cc -target ${TARGET}" +export CXX="zig c++ -target ${TARGET}" +export CHOST=${ARCH} echo "::endgroup::" ######## @@ -367,4 +358,4 @@ mv python-install python-${PYTHON_FULL_VER}-linux-${ARCH} tar -czf ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz python-${PYTHON_FULL_VER}-linux-${ARCH} zip ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.zip $(tar tf ${WORKDIR}/python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz) -echo "::endgroup::" \ No newline at end of file +echo "::endgroup::" From 81743e84128ba0724605b774ffb6eaef0c0e615e Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 21:59:11 -0500 Subject: [PATCH 061/161] zig tweaks --- .github/workflows/build_python_linux.yml | 2 +- zigshim/zig_ar | 2 -- zigshim/zig_cc | 2 -- zigshim/zig_cxx | 2 -- 4 files changed, 1 insertion(+), 7 deletions(-) delete mode 100755 zigshim/zig_ar delete mode 100755 zigshim/zig_cc delete mode 100755 zigshim/zig_cxx diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index f31a4b87..58a42d22 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -35,7 +35,7 @@ jobs: - name: Build run: | - sudo ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} + sudo -E ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/zigshim/zig_ar b/zigshim/zig_ar deleted file mode 100755 index 80b2e84f..00000000 --- a/zigshim/zig_ar +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -zig ar "$@" \ No newline at end of file diff --git a/zigshim/zig_cc b/zigshim/zig_cc deleted file mode 100755 index 5e5302c2..00000000 --- a/zigshim/zig_cc +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -zig cc -target $ZIG_TARGET "$@" \ No newline at end of file diff --git a/zigshim/zig_cxx b/zigshim/zig_cxx deleted file mode 100755 index 15fd2e9f..00000000 --- a/zigshim/zig_cxx +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -zig c++ -target $ZIG_TARGET "$@" \ No newline at end of file From e36c159788384ed4b6e93b3579a33d998a0a57f8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 22:03:05 -0500 Subject: [PATCH 062/161] Revert "zig tweaks" This reverts commit 81743e84128ba0724605b774ffb6eaef0c0e615e. --- .github/workflows/build_python_linux.yml | 2 +- zigshim/zig_ar | 2 ++ zigshim/zig_cc | 2 ++ zigshim/zig_cxx | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) create mode 100755 zigshim/zig_ar create mode 100755 zigshim/zig_cc create mode 100755 zigshim/zig_cxx diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index 58a42d22..f31a4b87 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -35,7 +35,7 @@ jobs: - name: Build run: | - sudo -E ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} + sudo ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/zigshim/zig_ar b/zigshim/zig_ar new file mode 100755 index 00000000..80b2e84f --- /dev/null +++ b/zigshim/zig_ar @@ -0,0 +1,2 @@ +#!/bin/bash +zig ar "$@" \ No newline at end of file diff --git a/zigshim/zig_cc b/zigshim/zig_cc new file mode 100755 index 00000000..5e5302c2 --- /dev/null +++ b/zigshim/zig_cc @@ -0,0 +1,2 @@ +#!/bin/bash +zig cc -target $ZIG_TARGET "$@" \ No newline at end of file diff --git a/zigshim/zig_cxx b/zigshim/zig_cxx new file mode 100755 index 00000000..15fd2e9f --- /dev/null +++ b/zigshim/zig_cxx @@ -0,0 +1,2 @@ +#!/bin/bash +zig c++ -target $ZIG_TARGET "$@" \ No newline at end of file From e6e06fb4eceb41e6a76ff97a4dffa7ff283ef1b2 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 22:06:16 -0500 Subject: [PATCH 063/161] zig tweaks --- scripts/build_linux_zig.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 5d97af7d..b598b9ab 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -41,9 +41,9 @@ mkdir ${DEPSDIR} export TARGET=${ARCH}-linux-gnu.2.17 -export AR="zig ar" -export CC="zig cc -target ${TARGET}" -export CXX="zig c++ -target ${TARGET}" +export AR="${WORKDIR}/zigshim/zig_ar" +export CC="${WORKDIR}/zigshim/zig_cc" +export CXX="${WORKDIR}/zigshim/zig_cxx" export CHOST=${ARCH} echo "::endgroup::" From d194b8a27d5a0999c3e455331f8f8b735b8f0d45 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 22:09:38 -0500 Subject: [PATCH 064/161] zig version --- scripts/build_linux_zig.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index b598b9ab..74da242c 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -5,6 +5,7 @@ PYTHON_FULL_VER=$2 PYTHON_VER=$(echo ${PYTHON_FULL_VER} | cut -d "." -f 1-2) set -ex +zig version WORKDIR=$(pwd) BUILDDIR=${WORKDIR}/build From 914cfcb50c8cd9cd5877f904c2deb9602bc6b3b7 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 22:14:16 -0500 Subject: [PATCH 065/161] sudo fixes --- .github/workflows/build_python_linux.yml | 2 +- scripts/build_linux_zig.sh | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index f31a4b87..d64a19c8 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -35,7 +35,7 @@ jobs: - name: Build run: | - sudo ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} + ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 74da242c..36f151ae 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -19,23 +19,23 @@ trap "cd ${BUILDDIR}/python-build && tar -czf ${WORKDIR}/build-python-${PYTHON_F echo "::group::Install dependencies" export DEBIAN_FRONTEND=noninteractive -apt update -apt -y install wget build-essential pkg-config cmake autoconf git python3 python3-pip clang patchelf qemu-user-static +sudo apt update +sudo apt -y install wget build-essential pkg-config cmake autoconf git python3 python3-pip clang patchelf qemu-user-static case "$ARCH" in x86_64) - apt -y install libc6-amd64-cross - ln -s /usr/x86_64-linux-gnu/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 + sudo apt -y install libc6-amd64-cross + sudo ln -s /usr/x86_64-linux-gnu/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 ;; aarch64) - apt -y install libc6-arm64-cross - ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/ld-linux-aarch64.so.1 + sudo apt -y install libc6-arm64-cross + sudo ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/ld-linux-aarch64.so.1 ;; arm) - apt -y install libc6-armhf-cross - ln -s /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3 + sudo apt -y install libc6-armhf-cross + sudo ln -s /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3 ;; esac -pip install https://github.com/mesonbuild/meson/archive/2baae24.zip ninja +sudo pip install https://github.com/mesonbuild/meson/archive/2baae24.zip ninja mkdir ${BUILDDIR} mkdir ${DEPSDIR} From 6efd7c7ccbde75432f12ad7c2786c9fcd7ab054b Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 22:15:51 -0500 Subject: [PATCH 066/161] Revert "zig tweaks" This reverts commit e6e06fb4eceb41e6a76ff97a4dffa7ff283ef1b2. --- scripts/build_linux_zig.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 36f151ae..6e12fa27 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -42,9 +42,9 @@ mkdir ${DEPSDIR} export TARGET=${ARCH}-linux-gnu.2.17 -export AR="${WORKDIR}/zigshim/zig_ar" -export CC="${WORKDIR}/zigshim/zig_cc" -export CXX="${WORKDIR}/zigshim/zig_cxx" +export AR="zig ar" +export CC="zig cc -target ${TARGET}" +export CXX="zig c++ -target ${TARGET}" export CHOST=${ARCH} echo "::endgroup::" From 576fa7650674405efb972e0e47607371a87767cd Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 22:18:00 -0500 Subject: [PATCH 067/161] remove shim --- zigshim/zig_ar | 2 -- zigshim/zig_cc | 2 -- zigshim/zig_cxx | 2 -- 3 files changed, 6 deletions(-) delete mode 100755 zigshim/zig_ar delete mode 100755 zigshim/zig_cc delete mode 100755 zigshim/zig_cxx diff --git a/zigshim/zig_ar b/zigshim/zig_ar deleted file mode 100755 index 80b2e84f..00000000 --- a/zigshim/zig_ar +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -zig ar "$@" \ No newline at end of file diff --git a/zigshim/zig_cc b/zigshim/zig_cc deleted file mode 100755 index 5e5302c2..00000000 --- a/zigshim/zig_cc +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -zig cc -target $ZIG_TARGET "$@" \ No newline at end of file diff --git a/zigshim/zig_cxx b/zigshim/zig_cxx deleted file mode 100755 index 15fd2e9f..00000000 --- a/zigshim/zig_cxx +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -zig c++ -target $ZIG_TARGET "$@" \ No newline at end of file From 222a6625a80bd8a46e2ad56aae4507d11960efef Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 23:07:27 -0500 Subject: [PATCH 068/161] add option to run tests --- .github/workflows/build_python.yml | 17 +++++++++++++---- .github/workflows/build_python_linux.yml | 13 +++++++++---- .github/workflows/build_python_on_branch.yml | 4 ++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index d28fa6a9..26d3f515 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -6,11 +6,17 @@ on: python_version: required: true type: string + run_tests: + required: false + type: boolean workflow_call: inputs: python_version: required: true type: string + run_tests: + required: false + type: boolean jobs: build_linux: @@ -23,6 +29,7 @@ jobs: with: python_version: ${{ inputs.python_version }} arch: ${{ matrix.arch }} + run_tests: ${{ inputs.run_tests }} build_windows: name: Windows ${{ inputs.python_version }} x86_64 (build) @@ -80,8 +87,9 @@ jobs: bin/python --version ./bin/python -c "import ssl" - # make tests nonfatal for now - # ./bin/python -m test || true + if [[ "${{ inputs.run_tests }}" == "true" ]]; then + ./bin/python -m test + fi build_macos: name: MacOS ${{ inputs.python_version }} universal2 (build) @@ -131,5 +139,6 @@ jobs: ./bin/python --version ./bin/python -c "import ssl" - # make tests nonfatal for now - # ./bin/python -m test || true + if [[ "${{ inputs.run_tests }}" == "true" ]]; then + ./bin/python -m test + fi diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index c73d5a2b..b7343b05 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -9,6 +9,9 @@ on: arch: required: true type: string + run_tests: + required: false + type: boolean env: image_map: '{"x86_64": "debian:buster", "aarch64": "debian:buster", "armv7l": "debian:buster"}' @@ -72,8 +75,9 @@ jobs: ./bin/python --version ./bin/python -c "import ssl" - # make tests nonfatal for now - # ./bin/python -m test || true + if [[ "${{ inputs.run_tests }}" == "true" ]]; then + ./bin/python -m test + fi build_linux: if: ${{ startsWith(inputs.python_version, '3.8') || inputs.arch == 'armv7l' }} @@ -141,5 +145,6 @@ jobs: ./bin/python --version ./bin/python -c "import ssl" - # make tests nonfatal for now - # ./bin/python -m test || true \ No newline at end of file + if [[ "${{ inputs.run_tests }}" == "true" ]]; then + ./bin/python -m test + fi \ No newline at end of file diff --git a/.github/workflows/build_python_on_branch.yml b/.github/workflows/build_python_on_branch.yml index 41383fbd..7ac699e8 100644 --- a/.github/workflows/build_python_on_branch.yml +++ b/.github/workflows/build_python_on_branch.yml @@ -7,6 +7,10 @@ on: pull_request: paths: ["scripts/**", ".github/workflows/**"] workflow_dispatch: + inputs: + run_tests: + required: false + type: boolean jobs: build: From 1a3e6744e63ac6d1d8b9199341c8cd43e0470987 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 23:18:04 -0500 Subject: [PATCH 069/161] propagate test flag --- .github/workflows/build_python_on_branch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_python_on_branch.yml b/.github/workflows/build_python_on_branch.yml index 7ac699e8..4956630a 100644 --- a/.github/workflows/build_python_on_branch.yml +++ b/.github/workflows/build_python_on_branch.yml @@ -22,3 +22,4 @@ jobs: uses: ./.github/workflows/build_python.yml with: python_version: ${{ matrix.python_version }} + run_tests: ${{ inputs.run_tests }} From 63aa3471fe5576a801960a046e87f127041a04ac Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 23:19:27 -0500 Subject: [PATCH 070/161] set default --- .github/workflows/build_python_on_branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_python_on_branch.yml b/.github/workflows/build_python_on_branch.yml index 4956630a..05d318b8 100644 --- a/.github/workflows/build_python_on_branch.yml +++ b/.github/workflows/build_python_on_branch.yml @@ -22,4 +22,4 @@ jobs: uses: ./.github/workflows/build_python.yml with: python_version: ${{ matrix.python_version }} - run_tests: ${{ inputs.run_tests }} + run_tests: ${{ inputs.run_tests || false }} From ce5d71cbdc065226d6ff8f9772fefe45cad4ec3c Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 23:34:14 -0500 Subject: [PATCH 071/161] build extensions as builtin --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 6e12fa27..51cce336 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -254,7 +254,7 @@ CFLAGS="-I${DEPSDIR}/include" cmake \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ - -DBUILD_EXTENSIONS_AS_BUILTIN=OFF \ + -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ -DBUILD_LIBPYTHON_SHARED=ON \ -DUSE_SYSTEM_LIBRARIES=OFF \ -DBUILD_TESTING=ON \ From 4b356239a570e30e842ba8859cd4fc60383420d4 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 23:43:18 -0500 Subject: [PATCH 072/161] use shared libs instead --- scripts/build_linux_zig.sh | 43 ++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 51cce336..8f25d70d 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -73,7 +73,7 @@ wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar. tar -xf OpenSSL*.tar.gz rm *.tar.gz cd openssl-OpenSSL* -./Configure linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} +./Configure shared linux-${ARCH} --prefix=${DEPSDIR} --openssldir=${DEPSDIR} make -j4 make install_sw @@ -184,7 +184,7 @@ rm *.tar.gz cd libuuid* mkdir build cd build -cmake -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. +cmake -DCMAKE_LINKER=ld -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=ON -DLIBUUID_STATIC=ON .. make -j4 make install @@ -233,6 +233,20 @@ cd ${BUILDDIR} #make -j4 #make install +echo "::endgroup::" +############# +# mpdecimal # +############# +echo "::group::mpdecimal" +cd ${BUILDDIR} + +wget -q https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.0.tar.gz +tar -xzf mpdecimal*.tar.gz +cd mpdecimal-2.5.0 +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +make -j4 +make install + echo "::endgroup::" ########## # Python # @@ -247,7 +261,7 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -CFLAGS="-I${DEPSDIR}/include" cmake \ +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_C_STANDARD=99 \ @@ -256,31 +270,10 @@ CFLAGS="-I${DEPSDIR}/include" cmake \ -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ -DBUILD_LIBPYTHON_SHARED=ON \ - -DUSE_SYSTEM_LIBRARIES=OFF \ + -DUSE_SYSTEM_LIBMPDEC=ON \ -DBUILD_TESTING=ON \ -DINSTALL_TEST=OFF \ -DINSTALL_MANUAL=OFF \ - -DOPENSSL_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DOPENSSL_LIBRARIES="${DEPSDIR}/lib/libssl.a;${DEPSDIR}/lib/libcrypto.a" \ - -DSQLite3_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DSQLite3_LIBRARY:FILEPATH=${DEPSDIR}/lib/libsqlite3.a \ - -DZLIB_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DZLIB_LIBRARY:FILEPATH=${DEPSDIR}/lib/libz.a \ - -DLZMA_INCLUDE_PATH:PATH=${DEPSDIR}/include \ - -DLZMA_LIBRARY:FILEPATH=${DEPSDIR}/lib/liblzma.a \ - -DBZIP2_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DBZIP2_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libbz2.a \ - -DLibFFI_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DLibFFI_LIBRARY:FILEPATH=${DEPSDIR}/lib/libffi.a \ - -DREADLINE_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/readline/readline.h \ - -DREADLINE_LIBRARY:FILEPATH=${DEPSDIR}/lib/libreadline.a \ - -DUUID_LIBRARY:FILEPATH=${DEPSDIR}/lib/libuuid_static.a \ - -DCURSES_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libncurses.a \ - -DPANEL_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libpanel.a \ - -DGDBM_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/gdbm.h \ - -DGDBM_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm.a \ - -DGDBM_COMPAT_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm_compat.a \ - -DNDBM_TAG=NDBM \ ../python-cmake-buildsystem make -j4 make install From 0974d57942e2e1488c996492c8fc1b0f1481efad Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 23:49:33 -0500 Subject: [PATCH 073/161] install tests if requested --- .github/workflows/build_python.yml | 3 +++ .github/workflows/build_python_linux.yml | 1 + scripts/build_linux.sh | 11 ++++++-- scripts/build_linux_zig.sh | 32 ++++++++++++++---------- scripts/build_macos.sh | 10 ++++++-- scripts/build_windows.sh | 10 ++++++-- 6 files changed, 48 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 26d3f515..b942e75b 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -18,6 +18,9 @@ on: required: false type: boolean +env: + RUN_TESTS: ${{ inputs.run_tests }} + jobs: build_linux: name: Linux ${{ inputs.python_version }} ${{ matrix.arch }} diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index da35f617..5b1c49f7 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -16,6 +16,7 @@ on: env: image_map: '{"x86_64": "debian:buster", "aarch64": "debian:buster", "armv7l": "debian:buster"}' platform_map: '{"x86_64": "linux/amd64", "aarch64": "linux/arm64/v8", "armv7l": "linux/arm/v7"}' + RUN_TESTS: ${{ inputs.run_tests }} jobs: build_linux_zig: diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh index 64d65867..41f81974 100755 --- a/scripts/build_linux.sh +++ b/scripts/build_linux.sh @@ -85,6 +85,13 @@ if [[ "${ARCH}" == "armv7l" ]]; then additionalparams+=(-DUSE_SYSTEM_LIBMPDEC=ON) export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib fi + +if [[ "${RUN_TESTS}" == "true" ]]; then + INSTALL_TEST="ON" +else + INSTALL_TEST="OFF" +fi + #cmake --trace-expand \ cmake \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ @@ -93,8 +100,8 @@ cmake \ -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ -DBUILD_LIBPYTHON_SHARED=ON \ "${additionalparams[@]}" \ - -DBUILD_TESTING=ON \ - -DINSTALL_TEST=OFF \ + -DBUILD_TESTING=${RUN_TESTS} \ + -DINSTALL_TEST=${RUN_TESTS} \ -DINSTALL_MANUAL=OFF \ ../python-cmake-buildsystem make -j8 diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 8f25d70d..869088de 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -254,6 +254,12 @@ echo "::endgroup::" echo "::group::Python" cd ${BUILDDIR} +if [[ "${RUN_TESTS}" == "true" ]]; then + INSTALL_TEST="ON" +else + INSTALL_TEST="OFF" +fi + wget -q -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/portable-python tar -xf python-cmake-buildsystem.tar.gz rm *.tar.gz @@ -262,19 +268,19 @@ mkdir python-build mkdir python-install cd python-build CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" cmake \ - -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ - -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ - -DCMAKE_C_STANDARD=99 \ - -DPYTHON_VERSION=${PYTHON_FULL_VER} \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ - -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ - -DBUILD_LIBPYTHON_SHARED=ON \ - -DUSE_SYSTEM_LIBMPDEC=ON \ - -DBUILD_TESTING=ON \ - -DINSTALL_TEST=OFF \ - -DINSTALL_MANUAL=OFF \ - ../python-cmake-buildsystem + -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ + -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ + -DCMAKE_C_STANDARD=99 \ + -DPYTHON_VERSION=${PYTHON_FULL_VER} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ + -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ + -DBUILD_LIBPYTHON_SHARED=ON \ + -DUSE_SYSTEM_LIBMPDEC=ON \ + -DBUILD_TESTING=${INSTALL_TEST} \ + -DINSTALL_TEST=${INSTALL_TEST} \ + -DINSTALL_MANUAL=OFF \ + ../python-cmake-buildsystem make -j4 make install diff --git a/scripts/build_macos.sh b/scripts/build_macos.sh index 75d70535..9e4093d5 100755 --- a/scripts/build_macos.sh +++ b/scripts/build_macos.sh @@ -168,6 +168,12 @@ echo "::endgroup::" echo "::group::Build" cd ${WORKDIR} +if [[ "${RUN_TESTS}" == "true" ]]; then + INSTALL_TEST="ON" +else + INSTALL_TEST="OFF" +fi + cd python-build cmake \ -G "Unix Makefiles" \ @@ -179,8 +185,8 @@ cmake \ -DCMAKE_INSTALL_PREFIX:PATH=${WORKDIR}/python-install \ -DBUILD_EXTENSIONS_AS_BUILTIN=OFF \ -DBUILD_LIBPYTHON_SHARED=ON \ - -DBUILD_TESTING=ON \ - -DINSTALL_TEST=OFF \ + -DBUILD_TESTING=${INSTALL_TEST} \ + -DINSTALL_TEST=${INSTALL_TEST} \ -DINSTALL_MANUAL=OFF \ -DOPENSSL_ROOT_DIR:PATH=${WORKDIR}/deps/openssl \ -DSQLite3_INCLUDE_DIR:PATH=${WORKDIR}/deps/sqlite3/include \ diff --git a/scripts/build_windows.sh b/scripts/build_windows.sh index 7a38e8cc..d13fca4d 100755 --- a/scripts/build_windows.sh +++ b/scripts/build_windows.sh @@ -127,6 +127,12 @@ echo "::endgroup::" echo "::group::Build" cd ${WORKDIR} +if [[ "${RUN_TESTS}" == "true" ]]; then + INSTALL_TEST="ON" +else + INSTALL_TEST="OFF" +fi + cd python-build cmake \ -G "Visual Studio 17 2022" -A x64 \ @@ -136,8 +142,8 @@ cmake \ -DCMAKE_INSTALL_PREFIX:PATH=${WORKDIR}/python-install \ -DBUILD_EXTENSIONS_AS_BUILTIN=OFF \ -DBUILD_LIBPYTHON_SHARED=ON \ - -DBUILD_TESTING=ON \ - -DINSTALL_TEST=OFF \ + -DBUILD_TESTING=${INSTALL_TEST} \ + -DINSTALL_TEST=${INSTALL_TEST} \ -DINSTALL_MANUAL=OFF \ -DBUILD_WININST=OFF \ -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF \ From a1f5a24a17cf32e1c99858c060b51b981920f7eb Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 23:57:53 -0500 Subject: [PATCH 074/161] fix vars --- scripts/build_linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh index 41f81974..a02e1dd7 100755 --- a/scripts/build_linux.sh +++ b/scripts/build_linux.sh @@ -100,8 +100,8 @@ cmake \ -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ -DBUILD_LIBPYTHON_SHARED=ON \ "${additionalparams[@]}" \ - -DBUILD_TESTING=${RUN_TESTS} \ - -DINSTALL_TEST=${RUN_TESTS} \ + -DBUILD_TESTING=${INSTALL_TEST} \ + -DINSTALL_TEST=${INSTALL_TEST} \ -DINSTALL_MANUAL=OFF \ ../python-cmake-buildsystem make -j8 From 30d71ed2e32872ea70eecf2b99e5ea7316924d71 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 27 Jan 2024 23:58:15 -0500 Subject: [PATCH 075/161] disable libuuid shared --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 869088de..0fe447a3 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -184,7 +184,7 @@ rm *.tar.gz cd libuuid* mkdir build cd build -cmake -DCMAKE_LINKER=ld -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=ON -DLIBUUID_STATIC=ON .. +cmake -DCMAKE_LINKER=ld -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. make -j4 make install From 789aea3cd531eae228a2a49b20736841c4e7af12 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 00:15:14 -0500 Subject: [PATCH 076/161] use zig ranlib --- scripts/build_linux_zig.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 0fe447a3..f1830dee 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -43,6 +43,7 @@ mkdir ${DEPSDIR} export TARGET=${ARCH}-linux-gnu.2.17 export AR="zig ar" +export RANLIB="zig ranlib" export CC="zig cc -target ${TARGET}" export CXX="zig c++ -target ${TARGET}" export CHOST=${ARCH} From d317a19dad8071a06ed78cf26a6ca1f98b3b5f7d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 00:16:22 -0500 Subject: [PATCH 077/161] disable libmpdec++ --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index f1830dee..54ccaf1e 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -244,7 +244,7 @@ cd ${BUILDDIR} wget -q https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.0.tar.gz tar -xzf mpdecimal*.tar.gz cd mpdecimal-2.5.0 -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --disable-cxx --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From f893e2348b1f0b6afcc91879d2572f2be3338e7d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 00:27:25 -0500 Subject: [PATCH 078/161] set cmake search path --- scripts/build_linux_zig.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 54ccaf1e..029b8f91 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -272,6 +272,7 @@ CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_C_STANDARD=99 \ + -DCMAKE_LIBRARY_PATH=${DEPSDIR}/lib \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ From 0e95c9766776669ee8b5a6e9289a5bd5b7882959 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 08:42:21 -0500 Subject: [PATCH 079/161] Revert "remove shim" This reverts commit 576fa7650674405efb972e0e47607371a87767cd. --- zigshim/zig_ar | 2 ++ zigshim/zig_cc | 2 ++ zigshim/zig_cxx | 2 ++ 3 files changed, 6 insertions(+) create mode 100755 zigshim/zig_ar create mode 100755 zigshim/zig_cc create mode 100755 zigshim/zig_cxx diff --git a/zigshim/zig_ar b/zigshim/zig_ar new file mode 100755 index 00000000..80b2e84f --- /dev/null +++ b/zigshim/zig_ar @@ -0,0 +1,2 @@ +#!/bin/bash +zig ar "$@" \ No newline at end of file diff --git a/zigshim/zig_cc b/zigshim/zig_cc new file mode 100755 index 00000000..5e5302c2 --- /dev/null +++ b/zigshim/zig_cc @@ -0,0 +1,2 @@ +#!/bin/bash +zig cc -target $ZIG_TARGET "$@" \ No newline at end of file diff --git a/zigshim/zig_cxx b/zigshim/zig_cxx new file mode 100755 index 00000000..15fd2e9f --- /dev/null +++ b/zigshim/zig_cxx @@ -0,0 +1,2 @@ +#!/bin/bash +zig c++ -target $ZIG_TARGET "$@" \ No newline at end of file From 46bdd150c04be3ffc3f1fb8034b4c9425966331e Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 08:46:19 -0500 Subject: [PATCH 080/161] back to shims --- scripts/build_linux_zig.sh | 8 ++++---- zigshim/zig_ranlib | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100755 zigshim/zig_ranlib diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 029b8f91..f09ee436 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -42,10 +42,10 @@ mkdir ${DEPSDIR} export TARGET=${ARCH}-linux-gnu.2.17 -export AR="zig ar" -export RANLIB="zig ranlib" -export CC="zig cc -target ${TARGET}" -export CXX="zig c++ -target ${TARGET}" +export AR="${WORKDIR}/zigshim/zig_ar" +export RANLIB="${WORKDIR}/zigshim/zig_ranlib" +export CC="${WORKDIR}/zigshim/zig_cc" +export CXX="${WORKDIR}/zigshim/zig_cxx" export CHOST=${ARCH} echo "::endgroup::" diff --git a/zigshim/zig_ranlib b/zigshim/zig_ranlib new file mode 100755 index 00000000..32c16893 --- /dev/null +++ b/zigshim/zig_ranlib @@ -0,0 +1,2 @@ +#!/bin/bash +zig ranlib "$@" From a39517785382a05e556c62a0c69e75a7ca21982a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 08:49:35 -0500 Subject: [PATCH 081/161] fix env silliness --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index f09ee436..b87c8a0d 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -40,7 +40,7 @@ sudo pip install https://github.com/mesonbuild/meson/archive/2baae24.zip ninja mkdir ${BUILDDIR} mkdir ${DEPSDIR} -export TARGET=${ARCH}-linux-gnu.2.17 +export ZIG_TARGET=${ARCH}-linux-gnu.2.17 export AR="${WORKDIR}/zigshim/zig_ar" export RANLIB="${WORKDIR}/zigshim/zig_ranlib" From 6d6f4f21c10a29bbf6549bef8752715c9376b59c Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 09:17:59 -0500 Subject: [PATCH 082/161] Revert "use shared libs instead" This reverts commit 4b356239a570e30e842ba8859cd4fc60383420d4. --- scripts/build_linux_zig.sh | 57 ++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index b87c8a0d..276ff84e 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -74,7 +74,7 @@ wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar. tar -xf OpenSSL*.tar.gz rm *.tar.gz cd openssl-OpenSSL* -./Configure shared linux-${ARCH} --prefix=${DEPSDIR} --openssldir=${DEPSDIR} +./Configure linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} make -j4 make install_sw @@ -185,7 +185,7 @@ rm *.tar.gz cd libuuid* mkdir build cd build -cmake -DCMAKE_LINKER=ld -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. +cmake -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. make -j4 make install @@ -235,6 +235,7 @@ cd ${BUILDDIR} #make install echo "::endgroup::" +<<<<<<< HEAD ############# # mpdecimal # ############# @@ -249,6 +250,8 @@ make -j4 make install echo "::endgroup::" +======= +>>>>>>> parent of 4b35623 (use shared libs instead) ########## # Python # ########## @@ -268,21 +271,41 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" cmake \ - -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ - -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ - -DCMAKE_C_STANDARD=99 \ - -DCMAKE_LIBRARY_PATH=${DEPSDIR}/lib \ - -DPYTHON_VERSION=${PYTHON_FULL_VER} \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ - -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ - -DBUILD_LIBPYTHON_SHARED=ON \ - -DUSE_SYSTEM_LIBMPDEC=ON \ - -DBUILD_TESTING=${INSTALL_TEST} \ - -DINSTALL_TEST=${INSTALL_TEST} \ - -DINSTALL_MANUAL=OFF \ - ../python-cmake-buildsystem +CFLAGS="-I${DEPSDIR}/include" cmake \ + -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ + -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ + -DCMAKE_C_STANDARD=99 \ + -DPYTHON_VERSION=${PYTHON_FULL_VER} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ + -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ + -DBUILD_LIBPYTHON_SHARED=ON \ + -DUSE_SYSTEM_LIBRARIES=OFF \ + -DBUILD_TESTING=ON \ + -DINSTALL_TEST=OFF \ + -DINSTALL_MANUAL=OFF \ + -DOPENSSL_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DOPENSSL_LIBRARIES="${DEPSDIR}/lib/libssl.a;${DEPSDIR}/lib/libcrypto.a" \ + -DSQLite3_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DSQLite3_LIBRARY:FILEPATH=${DEPSDIR}/lib/libsqlite3.a \ + -DZLIB_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DZLIB_LIBRARY:FILEPATH=${DEPSDIR}/lib/libz.a \ + -DLZMA_INCLUDE_PATH:PATH=${DEPSDIR}/include \ + -DLZMA_LIBRARY:FILEPATH=${DEPSDIR}/lib/liblzma.a \ + -DBZIP2_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DBZIP2_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libbz2.a \ + -DLibFFI_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DLibFFI_LIBRARY:FILEPATH=${DEPSDIR}/lib/libffi.a \ + -DREADLINE_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/readline/readline.h \ + -DREADLINE_LIBRARY:FILEPATH=${DEPSDIR}/lib/libreadline.a \ + -DUUID_LIBRARY:FILEPATH=${DEPSDIR}/lib/libuuid_static.a \ + -DCURSES_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libncurses.a \ + -DPANEL_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libpanel.a \ + -DGDBM_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/gdbm.h \ + -DGDBM_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm.a \ + -DGDBM_COMPAT_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm_compat.a \ + -DNDBM_TAG=NDBM \ + ../python-cmake-buildsystem make -j4 make install From f8d85c1910a0ed38cccea6fae3c4f6be9dfd74d8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 09:41:16 -0500 Subject: [PATCH 083/161] fix missed merge conflict --- scripts/build_linux_zig.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 276ff84e..7e1bf5ba 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -235,7 +235,6 @@ cd ${BUILDDIR} #make install echo "::endgroup::" -<<<<<<< HEAD ############# # mpdecimal # ############# @@ -250,8 +249,6 @@ make -j4 make install echo "::endgroup::" -======= ->>>>>>> parent of 4b35623 (use shared libs instead) ########## # Python # ########## From 31cbb2dd9d78112438c3131b8fc6574a211f6c5e Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 10:49:54 -0500 Subject: [PATCH 084/161] patch librt --- scripts/qemu_aarch64_interpreter | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/qemu_aarch64_interpreter b/scripts/qemu_aarch64_interpreter index 6363988b..0968b059 100755 --- a/scripts/qemu_aarch64_interpreter +++ b/scripts/qemu_aarch64_interpreter @@ -9,6 +9,7 @@ patchelf --add-needed libm.so.6 $1 patchelf --add-needed libpthread.so.0 $1 patchelf --add-needed libdl.so.2 $1 patchelf --add-needed libutil.so.1 $1 +patchelf --add-needed librt.so.1 $1 # run the interpreter -qemu-aarch64-static "$@" \ No newline at end of file +qemu-aarch64-static "$@" From 80f84acab0669817c7570b20af5b9a5185686292 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 11:09:11 -0500 Subject: [PATCH 085/161] build tests --- scripts/build_linux_zig.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 7e1bf5ba..6cf95d82 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -278,8 +278,8 @@ CFLAGS="-I${DEPSDIR}/include" cmake \ -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ -DBUILD_LIBPYTHON_SHARED=ON \ -DUSE_SYSTEM_LIBRARIES=OFF \ - -DBUILD_TESTING=ON \ - -DINSTALL_TEST=OFF \ + -DBUILD_TESTING=${INSTALL_TEST} \ + -DINSTALL_TEST=${INSTALL_TEST} \ -DINSTALL_MANUAL=OFF \ -DOPENSSL_INCLUDE_DIR:PATH=${DEPSDIR}/include \ -DOPENSSL_LIBRARIES="${DEPSDIR}/lib/libssl.a;${DEPSDIR}/lib/libcrypto.a" \ From ed3f4618f6208e739dbc07e0e4773611ef678b1c Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 16:07:57 -0500 Subject: [PATCH 086/161] tweak test suite invocation --- .github/workflows/build_python.yml | 4 ++-- .github/workflows/build_python_linux.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index b942e75b..3080598f 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -91,7 +91,7 @@ jobs: ./bin/python -c "import ssl" if [[ "${{ inputs.run_tests }}" == "true" ]]; then - ./bin/python -m test + ./bin/python -m test -v -ulargefile,network,decimal,cpu,subprocess,urlfetch,tzdata fi build_macos: @@ -143,5 +143,5 @@ jobs: ./bin/python -c "import ssl" if [[ "${{ inputs.run_tests }}" == "true" ]]; then - ./bin/python -m test + ./bin/python -m test -v -ulargefile,network,decimal,cpu,subprocess,urlfetch,tzdata fi diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index 5b1c49f7..94aef3fc 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -80,7 +80,7 @@ jobs: ./bin/python -c "import ssl" if [[ "${{ inputs.run_tests }}" == "true" ]]; then - ./bin/python -m test + ./bin/python -m test -v -ulargefile,network,decimal,cpu,subprocess,urlfetch,tzdata fi build_linux: @@ -150,5 +150,5 @@ jobs: ./bin/python -c "import ssl" if [[ "${{ inputs.run_tests }}" == "true" ]]; then - ./bin/python -m test + ./bin/python -m test -v -ulargefile,network,decimal,cpu,subprocess,urlfetch,tzdata fi From c06b4357037cfac918935463be5d95bcfb6ee66e Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 16:43:26 -0500 Subject: [PATCH 087/161] fix indent --- scripts/build_linux_zig.sh | 68 +++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 6cf95d82..899c5423 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -269,40 +269,40 @@ mkdir python-build mkdir python-install cd python-build CFLAGS="-I${DEPSDIR}/include" cmake \ - -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ - -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ - -DCMAKE_C_STANDARD=99 \ - -DPYTHON_VERSION=${PYTHON_FULL_VER} \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ - -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ - -DBUILD_LIBPYTHON_SHARED=ON \ - -DUSE_SYSTEM_LIBRARIES=OFF \ - -DBUILD_TESTING=${INSTALL_TEST} \ - -DINSTALL_TEST=${INSTALL_TEST} \ - -DINSTALL_MANUAL=OFF \ - -DOPENSSL_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DOPENSSL_LIBRARIES="${DEPSDIR}/lib/libssl.a;${DEPSDIR}/lib/libcrypto.a" \ - -DSQLite3_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DSQLite3_LIBRARY:FILEPATH=${DEPSDIR}/lib/libsqlite3.a \ - -DZLIB_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DZLIB_LIBRARY:FILEPATH=${DEPSDIR}/lib/libz.a \ - -DLZMA_INCLUDE_PATH:PATH=${DEPSDIR}/include \ - -DLZMA_LIBRARY:FILEPATH=${DEPSDIR}/lib/liblzma.a \ - -DBZIP2_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DBZIP2_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libbz2.a \ - -DLibFFI_INCLUDE_DIR:PATH=${DEPSDIR}/include \ - -DLibFFI_LIBRARY:FILEPATH=${DEPSDIR}/lib/libffi.a \ - -DREADLINE_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/readline/readline.h \ - -DREADLINE_LIBRARY:FILEPATH=${DEPSDIR}/lib/libreadline.a \ - -DUUID_LIBRARY:FILEPATH=${DEPSDIR}/lib/libuuid_static.a \ - -DCURSES_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libncurses.a \ - -DPANEL_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libpanel.a \ - -DGDBM_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/gdbm.h \ - -DGDBM_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm.a \ - -DGDBM_COMPAT_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm_compat.a \ - -DNDBM_TAG=NDBM \ - ../python-cmake-buildsystem + -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ + -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ + -DCMAKE_C_STANDARD=99 \ + -DPYTHON_VERSION=${PYTHON_FULL_VER} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=${BUILDDIR}/python-install \ + -DBUILD_EXTENSIONS_AS_BUILTIN=ON \ + -DBUILD_LIBPYTHON_SHARED=ON \ + -DUSE_SYSTEM_LIBRARIES=OFF \ + -DBUILD_TESTING=${INSTALL_TEST} \ + -DINSTALL_TEST=${INSTALL_TEST} \ + -DINSTALL_MANUAL=OFF \ + -DOPENSSL_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DOPENSSL_LIBRARIES="${DEPSDIR}/lib/libssl.a;${DEPSDIR}/lib/libcrypto.a" \ + -DSQLite3_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DSQLite3_LIBRARY:FILEPATH=${DEPSDIR}/lib/libsqlite3.a \ + -DZLIB_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DZLIB_LIBRARY:FILEPATH=${DEPSDIR}/lib/libz.a \ + -DLZMA_INCLUDE_PATH:PATH=${DEPSDIR}/include \ + -DLZMA_LIBRARY:FILEPATH=${DEPSDIR}/lib/liblzma.a \ + -DBZIP2_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DBZIP2_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libbz2.a \ + -DLibFFI_INCLUDE_DIR:PATH=${DEPSDIR}/include \ + -DLibFFI_LIBRARY:FILEPATH=${DEPSDIR}/lib/libffi.a \ + -DREADLINE_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/readline/readline.h \ + -DREADLINE_LIBRARY:FILEPATH=${DEPSDIR}/lib/libreadline.a \ + -DUUID_LIBRARY:FILEPATH=${DEPSDIR}/lib/libuuid_static.a \ + -DCURSES_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libncurses.a \ + -DPANEL_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libpanel.a \ + -DGDBM_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/gdbm.h \ + -DGDBM_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm.a \ + -DGDBM_COMPAT_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm_compat.a \ + -DNDBM_TAG=NDBM \ + ../python-cmake-buildsystem make -j4 make install From c901fd06c15acd6ee76c7061a81aea0df1d70aa8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 16:51:36 -0500 Subject: [PATCH 088/161] fix sysconfig CC, etc. --- scripts/build_linux_zig.sh | 16 ++++++++++++---- zigshim/zig_ranlib | 2 -- 2 files changed, 12 insertions(+), 6 deletions(-) delete mode 100755 zigshim/zig_ranlib diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 899c5423..856cc0ba 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -42,10 +42,18 @@ mkdir ${DEPSDIR} export ZIG_TARGET=${ARCH}-linux-gnu.2.17 -export AR="${WORKDIR}/zigshim/zig_ar" -export RANLIB="${WORKDIR}/zigshim/zig_ranlib" -export CC="${WORKDIR}/zigshim/zig_cc" -export CXX="${WORKDIR}/zigshim/zig_cxx" +# Python's sysconfig module will retain references to these compiler values, which cause +# problems when sysconfig is used to pick a compiler during binary extension builds. +# Since clang (zig) is a drop-in replacement for gcc, we set these so the final sysconfig +# will work on other platforms. +export PATH="${WORKDIR}/zigshim:${PATH}" +cp ${WORKDIR}/zigshim/zig_ar ${WORKDIR}/zigshim/${ARCH}-linux-gnu-gcc-ar +cp ${WORKDIR}/zigshim/zig_cc ${WORKDIR}/zigshim/${ARCH}-linux-gnu-gcc +cp ${WORKDIR}/zigshim/zig_cxx ${WORKDIR}/zigshim/${ARCH}-linux-gnu-g++ + +export AR="${ARCH}-linux-gnu-gcc-ar" +export CC="${ARCH}-linux-gnu-gcc" +export CXX="${ARCH}-linux-gnu-g++" export CHOST=${ARCH} echo "::endgroup::" diff --git a/zigshim/zig_ranlib b/zigshim/zig_ranlib deleted file mode 100755 index 32c16893..00000000 --- a/zigshim/zig_ranlib +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -zig ranlib "$@" From a1f2cc187f103e096b409630c5b2510a6e85871f Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 16:56:59 -0500 Subject: [PATCH 089/161] bring back 3.8 --- .github/workflows/build_python_linux.yml | 4 ++-- .github/workflows/build_python_on_branch.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index 94aef3fc..fd329940 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -20,7 +20,7 @@ env: jobs: build_linux_zig: - if: ${{ !startsWith(inputs.python_version, '3.8') && inputs.arch != 'armv7l' }} + if: ${{ inputs.arch != 'armv7l' }} name: Linux (Zig) ${{ inputs.python_version }} ${{ inputs.arch }} runs-on: ubuntu-latest @@ -84,7 +84,7 @@ jobs: fi build_linux: - if: ${{ startsWith(inputs.python_version, '3.8') || inputs.arch == 'armv7l' }} + if: ${{ inputs.arch == 'armv7l' }} name: Linux (Docker) ${{ inputs.python_version }} ${{ inputs.arch }} runs-on: ubuntu-latest diff --git a/.github/workflows/build_python_on_branch.yml b/.github/workflows/build_python_on_branch.yml index 05d318b8..7ea3ce13 100644 --- a/.github/workflows/build_python_on_branch.yml +++ b/.github/workflows/build_python_on_branch.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: [ 3.9.17, 3.10.13 ] + python_version: [ 3.8.17, 3.9.17, 3.10.13 ] uses: ./.github/workflows/build_python.yml with: python_version: ${{ matrix.python_version }} From 13f9bafc2d5c19220d51de963017ba5385887a36 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 17:25:10 -0500 Subject: [PATCH 090/161] add libexpat --- scripts/build_linux_zig.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 856cc0ba..522b0d3a 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -116,6 +116,21 @@ cd sqlite* make -j4 make install +echo "::endgroup::" +######### +# expat # +######### +echo "::group::expat" +cd ${BUILDDIR} + +wget -q https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.gz +tar -xf expat*.tar.gz +rm *.tar.gz +cd expat* +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +make -j4 +make install + echo "::endgroup::" ############ # readline # From 7994fb60318e41684a148e790310451adb0098b5 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 17:41:33 -0500 Subject: [PATCH 091/161] heavy-handed overrides for sysconfig --- scripts/build_linux_zig.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 522b0d3a..b2713c62 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -46,10 +46,9 @@ export ZIG_TARGET=${ARCH}-linux-gnu.2.17 # problems when sysconfig is used to pick a compiler during binary extension builds. # Since clang (zig) is a drop-in replacement for gcc, we set these so the final sysconfig # will work on other platforms. -export PATH="${WORKDIR}/zigshim:${PATH}" -cp ${WORKDIR}/zigshim/zig_ar ${WORKDIR}/zigshim/${ARCH}-linux-gnu-gcc-ar -cp ${WORKDIR}/zigshim/zig_cc ${WORKDIR}/zigshim/${ARCH}-linux-gnu-gcc -cp ${WORKDIR}/zigshim/zig_cxx ${WORKDIR}/zigshim/${ARCH}-linux-gnu-g++ +sudo cp ${WORKDIR}/zigshim/zig_ar /usr/bin/${ARCH}-linux-gnu-gcc-ar +sudo cp ${WORKDIR}/zigshim/zig_cc /usr/bin/${ARCH}-linux-gnu-gcc +sudo cp ${WORKDIR}/zigshim/zig_cxx /usr/bin/${ARCH}-linux-gnu-g++ export AR="${ARCH}-linux-gnu-gcc-ar" export CC="${ARCH}-linux-gnu-gcc" From 2739a725e7e848f812201937652dedc01fe28d8c Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 28 Jan 2024 20:05:54 -0500 Subject: [PATCH 092/161] run test container using bash --- .github/workflows/build_python_linux.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index 5b1c49f7..16af84fb 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -59,6 +59,7 @@ jobs: with: image: centos:7 options: -v ${{ github.workspace }}:/work --workdir /tmp --platform ${{ steps.parse_platform.outputs.platform }} + shell: bash run: | set -e @@ -113,6 +114,7 @@ jobs: with: image: ${{ steps.parse_image.outputs.image }} options: -v ${{ github.workspace }}:/work --workdir /tmp --platform ${{ steps.parse_platform.outputs.platform }} + shell: bash run: | set -e /work/scripts/build_linux.sh ${{ inputs.arch }} ${{ inputs.python_version }} @@ -135,6 +137,7 @@ jobs: with: image: ${{ steps.parse_image.outputs.image }} options: -v ${{ github.workspace }}:/work --workdir /tmp --platform ${{ steps.parse_platform.outputs.platform }} + shell: bash run: | set -e apt update From a8dd285b0619bf11fa4509773a564daf5183380e Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 11:28:28 -0500 Subject: [PATCH 093/161] add x11 and deps --- scripts/build_linux_zig.sh | 135 +++++++++++++++++++++++++++++++------ scripts/x11_modfile.txt | 19 ++++++ 2 files changed, 132 insertions(+), 22 deletions(-) create mode 100644 scripts/x11_modfile.txt diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index b2713c62..b64bcd1e 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -20,7 +20,10 @@ echo "::group::Install dependencies" export DEBIAN_FRONTEND=noninteractive sudo apt update -sudo apt -y install wget build-essential pkg-config cmake autoconf git python3 python3-pip clang patchelf qemu-user-static +sudo apt -y install \ + wget build-essential pkg-config cmake autoconf git \ + python2 python3 python3-pip clang patchelf qemu-user-static \ + gettext bison libtool autopoint gperf case "$ARCH" in x86_64) sudo apt -y install libc6-amd64-cross @@ -201,13 +204,11 @@ echo "::endgroup::" echo "::group::uuid" cd ${BUILDDIR} -wget -q -O libuuid-cmake.tar.gz https://github.com/gershnik/libuuid-cmake/archive/refs/tags/v2.39.3.tar.gz -tar -xf libuuid*tar.gz -rm *.tar.gz -cd libuuid* -mkdir build -cd build -cmake -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} -DLIBUUID_SHARED=OFF -DLIBUUID_STATIC=ON .. +wget -q https://github.com/util-linux/util-linux/archive/refs/tags/v2.39.3.tar.gz +tar -xf *.tar.gz +cd util-linux* +./autogen.sh +./configure --disable-all-programs --enable-libuuid --prefix=${DEPSDIR} make -j4 make install @@ -226,6 +227,96 @@ cd gdbm* make -j4 make install +echo "::endgroup::" +########### +# libxml2 # +########### +echo "::group::libxml2" +cd ${BUILDDIR} + +wget -q https://download.gnome.org/sources/libxml2/2.12/libxml2-2.12.4.tar.xz +tar -xf libxml2*.tar.xz +rm *.tar.xz +cd libxml2* +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --without-python --prefix=${DEPSDIR} +make -j4 +make install + +echo "::endgroup::" +########### +# libxslt # +########### +echo "::group::libxslt" +cd ${BUILDDIR} + +wget -q https://download.gnome.org/sources/libxslt/1.1/libxslt-1.1.39.tar.xz +tar -xf libxslt*.tar.xz +rm *.tar.xz +cd libxslt* +CFLAGS="-I${DEPSDIR}/include -I${DEPSDIR}/include/libxml2" LDFLAGS="-L${DEPSDIR}/lib" ./configure --with-libxml-prefix=${DEPSDIR} --without-python --prefix=${DEPSDIR} +make -j4 +make install + +echo "::endgroup::" +############ +# freetype # +############ +echo "::group::freetype" +cd ${BUILDDIR} + +wget -q https://download.savannah.gnu.org/releases/freetype/freetype-2.13.2.tar.gz +tar -xf freetype*.tar.gz +rm *.tar.gz +cd freetype* +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --prefix=${DEPSDIR} +make -j4 +make install + +echo "::endgroup::" +############## +# fontconfig # +############## +echo "::group::fontconfig" +cd ${BUILDDIR} + +wget -q https://www.freedesktop.org/software/fontconfig/release/fontconfig-2.15.0.tar.gz +tar -xf fontconfig*.tar.gz +rm *.tar.gz +cd fontconfig* +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" PKG_CONFIG_PATH="${DEPSDIR}/lib/pkgconfig" ./configure --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} +make -j4 +make install + +echo "::endgroup::" +####### +# X11 # +####### +echo "::group::X11" +cd ${BUILDDIR} + +wget -q https://www.x.org/releases/individual/lib/libXau-1.0.11.tar.gz +wget -q https://www.x.org/releases/individual/lib/libXdmcp-1.1.2.tar.gz +wget -q https://www.x.org/releases/individual/lib/libX11-1.8.7.tar.gz +wget -q https://www.x.org/releases/individual/lib/libXext-1.3.5.tar.gz +wget -q https://www.x.org/releases/individual/lib/libICE-1.0.7.tar.gz +wget -q https://www.x.org/releases/individual/lib/libSM-1.2.2.tar.gz +wget -q https://www.x.org/releases/individual/lib/libXrender-0.9.11.tar.gz +wget -q https://www.x.org/releases/individual/lib/libXft-2.3.8.tar.gz +wget -q https://www.x.org/releases/individual/lib/libXScrnSaver-1.2.4.tar.gz +wget -q https://www.x.org/releases/individual/lib/xtrans-1.5.0.tar.gz +wget -q https://www.x.org/releases/individual/proto/xproto-7.0.31.tar.gz +wget -q https://www.x.org/releases/individual/proto/xextproto-7.3.0.tar.gz +wget -q https://www.x.org/releases/individual/proto/xcb-proto-1.16.0.tar.gz +wget -q https://www.x.org/releases/individual/proto/kbproto-1.0.7.tar.gz +wget -q https://www.x.org/releases/individual/proto/inputproto-2.3.2.tar.gz +wget -q https://www.x.org/releases/individual/proto/renderproto-0.11.1.tar.gz +wget -q https://www.x.org/releases/individual/proto/scrnsaverproto-1.2.2.tar.gz +wget -q https://www.x.org/releases/individual/xcb/libxcb-1.16.tar.gz +wget -q https://www.x.org/releases/individual/xcb/libpthread-stubs-0.5.tar.gz +git clone git://anongit.freedesktop.org/git/xorg/util/modular util/modular +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./util/modular/build.sh --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} +rm *.tar.gz + echo "::endgroup::" ####### # tcl # @@ -233,13 +324,13 @@ echo "::endgroup::" echo "::group::tcl" cd ${BUILDDIR} -#wget -q https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz -#tar -xf tcl*.tar.gz -#rm *.tar.gz -#cd tcl*/unix -#./configure --host=${ARCH}-linux --prefix=${DEPSDIR} -#make -j4 -#make install +wget -q https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz +tar -xf tcl*.tar.gz +rm *.tar.gz +cd tcl*/unix +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +make -j4 +make install echo "::endgroup::" ###### @@ -248,13 +339,13 @@ echo "::endgroup::" echo "::group::tk" cd ${BUILDDIR} -#wget -q https://prdownloads.sourceforge.net/tcl/tk8.6.13-src.tar.gz -#tar -xf tk*.tar.gz -#rm *.tar.gz -#cd tk*/unix -#CFLAGS="-I${DEPSDIR}/include" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} -#make -j4 -#make install +wget -q https://prdownloads.sourceforge.net/tcl/tk8.6.13-src.tar.gz +tar -xf tk*.tar.gz +rm *.tar.gz +cd tk*/unix +CFLAGS="-I${DEPSDIR}/include" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +make -j4 +make install echo "::endgroup::" ############# diff --git a/scripts/x11_modfile.txt b/scripts/x11_modfile.txt new file mode 100644 index 00000000..207a5371 --- /dev/null +++ b/scripts/x11_modfile.txt @@ -0,0 +1,19 @@ +proto/xproto +proto/xextproto +proto/kbproto +proto/inputproto +proto/renderproto +proto/scrnsaverproto +xcb/proto +xcb/pthread-stubs +lib/xtrans +lib/libXau +lib/libxcb +lib/libXdmcp +lib/libX11 +lib/libXext +lib/libICE +lib/libSM +lib/libXrender +lib/libXft +lib/libXScrnSaver From 4d0825c280a5d70f11be9883416d57a929c1d1d1 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 11:30:00 -0500 Subject: [PATCH 094/161] add x11 to cmake args --- scripts/build_linux_zig.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index b64bcd1e..ec388d60 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -408,13 +408,19 @@ CFLAGS="-I${DEPSDIR}/include" cmake \ -DLibFFI_LIBRARY:FILEPATH=${DEPSDIR}/lib/libffi.a \ -DREADLINE_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/readline/readline.h \ -DREADLINE_LIBRARY:FILEPATH=${DEPSDIR}/lib/libreadline.a \ - -DUUID_LIBRARY:FILEPATH=${DEPSDIR}/lib/libuuid_static.a \ + -DUUID_LIBRARY:FILEPATH=${DEPSDIR}/lib/libuuid.a \ -DCURSES_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libncurses.a \ -DPANEL_LIBRARIES:FILEPATH=${DEPSDIR}/lib/libpanel.a \ -DGDBM_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/gdbm.h \ -DGDBM_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm.a \ -DGDBM_COMPAT_LIBRARY:FILEPATH=${DEPSDIR}/lib/libgdbm_compat.a \ -DNDBM_TAG=NDBM \ + -DTK_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/tk.h \ + -DTK_LIBRARY:FILEPATH=${DEPSDIR}/lib/libtk8.6.a \ + -DTCL_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/tcl.h \ + -DTCL_LIBRARY:FILEPATH=${DEPSDIR}/lib/libtcl8.6.a \ + -DX11_INCLUDE_DIR:PATH=${DEPSDIR}/include/X11 \ + -DX11_LIBRARIES="${DEPSDIR}/lib/libXau.a;${DEPSDIR}/lib/libXdmcp.a;${DEPSDIR}/lib/libX11.a;${DEPSDIR}/lib/libXext.a;${DEPSDIR}/lib/libICE.a;${DEPSDIR}/lib/libSM.a;${DEPSDIR}/lib/libXrender.a;${DEPSDIR}/lib/libXft.a;${DEPSDIR}/lib/libXss.a" \ ../python-cmake-buildsystem make -j4 make install From ef6e441a70d88ac4c221b1322af413fa395f3c5b Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 11:41:06 -0500 Subject: [PATCH 095/161] add libgcrypt --- scripts/build_linux_zig.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index ec388d60..507e18e1 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -242,6 +242,21 @@ CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --without-p make -j4 make install +echo "::endgroup::" +############# +# libgcrypt # +############# +echo "::group::libgcrypt" +cd ${BUILDDIR} + +wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.11.tar.bz2 +tar -xf libgcrypt*.tar.bz2 +rm *.tar.bz2 +cd libgcrypt* +./configure --prefix=${DEPSDIR} +make -j4 +make install + echo "::endgroup::" ########### # libxslt # From ba2d2a6257478961024a7c21fb703d95bcf54ff5 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 11:49:28 -0500 Subject: [PATCH 096/161] add cross compile flags --- scripts/build_linux_zig.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 507e18e1..d7dc9698 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -208,7 +208,7 @@ wget -q https://github.com/util-linux/util-linux/archive/refs/tags/v2.39.3.tar.g tar -xf *.tar.gz cd util-linux* ./autogen.sh -./configure --disable-all-programs --enable-libuuid --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --disable-all-programs --enable-libuuid --prefix=${DEPSDIR} make -j4 make install @@ -238,7 +238,7 @@ wget -q https://download.gnome.org/sources/libxml2/2.12/libxml2-2.12.4.tar.xz tar -xf libxml2*.tar.xz rm *.tar.xz cd libxml2* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --without-python --prefix=${DEPSDIR} +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --without-python --prefix=${DEPSDIR} make -j4 make install @@ -253,7 +253,7 @@ wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.11.tar.bz2 tar -xf libgcrypt*.tar.bz2 rm *.tar.bz2 cd libgcrypt* -./configure --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -268,7 +268,7 @@ wget -q https://download.gnome.org/sources/libxslt/1.1/libxslt-1.1.39.tar.xz tar -xf libxslt*.tar.xz rm *.tar.xz cd libxslt* -CFLAGS="-I${DEPSDIR}/include -I${DEPSDIR}/include/libxml2" LDFLAGS="-L${DEPSDIR}/lib" ./configure --with-libxml-prefix=${DEPSDIR} --without-python --prefix=${DEPSDIR} +CFLAGS="-I${DEPSDIR}/include -I${DEPSDIR}/include/libxml2" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --with-libxml-prefix=${DEPSDIR} --without-python --prefix=${DEPSDIR} make -j4 make install @@ -283,7 +283,7 @@ wget -q https://download.savannah.gnu.org/releases/freetype/freetype-2.13.2.tar. tar -xf freetype*.tar.gz rm *.tar.gz cd freetype* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --prefix=${DEPSDIR} +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -298,7 +298,7 @@ wget -q https://www.freedesktop.org/software/fontconfig/release/fontconfig-2.15. tar -xf fontconfig*.tar.gz rm *.tar.gz cd fontconfig* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" PKG_CONFIG_PATH="${DEPSDIR}/lib/pkgconfig" ./configure --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" PKG_CONFIG_PATH="${DEPSDIR}/lib/pkgconfig" ./configure --host=${ARCH}-linux --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} make -j4 make install From 26cdf7b99918ac1de356ef38b5a95a80fc01e48a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 11:51:30 -0500 Subject: [PATCH 097/161] add libgpg-error --- scripts/build_linux_zig.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index d7dc9698..c9898a19 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -249,6 +249,16 @@ echo "::endgroup::" echo "::group::libgcrypt" cd ${BUILDDIR} +wget -q https://gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.47.tar.bz2 +tar -xf libgpg-error*.tar.bz2 +rm *.tar.bz2 +cd libgpg-error* +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +make -j4 +make install + +cd ${BUILDDIR} + wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.11.tar.bz2 tar -xf libgcrypt*.tar.bz2 rm *.tar.bz2 From 03015579f1fd4f3adf810bb68b786bab8da3550a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 12:01:09 -0500 Subject: [PATCH 098/161] cflags and ldflags --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index c9898a19..c25efa70 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -253,7 +253,7 @@ wget -q https://gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.47.tar.bz2 tar -xf libgpg-error*.tar.bz2 rm *.tar.bz2 cd libgpg-error* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From e7bc585f8bbbcd55ea8f0dc4d077791c73187b9a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 12:09:56 -0500 Subject: [PATCH 099/161] move cflags and ldflags --- scripts/build_linux_zig.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index c25efa70..4e821c57 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -253,7 +253,7 @@ wget -q https://gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.47.tar.bz2 tar -xf libgpg-error*.tar.bz2 rm *.tar.bz2 cd libgpg-error* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -263,7 +263,7 @@ wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.11.tar.bz2 tar -xf libgcrypt*.tar.bz2 rm *.tar.bz2 cd libgcrypt* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From f0d5976a3bc2353b527a4964d790721aa90de6b8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 12:20:12 -0500 Subject: [PATCH 100/161] newer libgcrypt --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 4e821c57..91f9918b 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -259,7 +259,7 @@ make install cd ${BUILDDIR} -wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.11.tar.bz2 +wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.10.3.tar.bz2 tar -xf libgcrypt*.tar.bz2 rm *.tar.bz2 cd libgcrypt* From b87f34de6b2b7d95537ef3e7bf9b33569d52443b Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 12:51:09 -0500 Subject: [PATCH 101/161] libgcrypt ldflags --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 91f9918b..698ae201 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -263,7 +263,7 @@ wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.10.3.tar.bz2 tar -xf libgcrypt*.tar.bz2 rm *.tar.bz2 cd libgcrypt* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib -Wl,--undefined-version" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From f7204fb5177e685ec9fa1aab7a37729e8ab6b1b2 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 13:00:56 -0500 Subject: [PATCH 102/161] disable libgcrypt asm --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 698ae201..808925c3 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -263,7 +263,7 @@ wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.10.3.tar.bz2 tar -xf libgcrypt*.tar.bz2 rm *.tar.bz2 cd libgcrypt* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib -Wl,--undefined-version" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib -Wl,--undefined-version" ./configure --disable-asm --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From de29b70b34328abf4f041efd9e84cb0d930b2120 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 13:06:08 -0500 Subject: [PATCH 103/161] add brotli --- scripts/build_linux_zig.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 808925c3..343290cb 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -197,6 +197,23 @@ cmake -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. make -j4 make install +echo "::endgroup::" +########## +# Brotli # +########## +echo "::group::Brotli" +cd ${BUILDDIR} + +wget -q https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz +tar -xf *.tar.gz +rm *.tar.gz +cd brotli* +mkdir build +cd build +cmake -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_INSTALL_PREFIX:PATH=${DEPSDIR} .. +make -j4 +make install + echo "::endgroup::" ######## # uuid # @@ -206,6 +223,7 @@ cd ${BUILDDIR} wget -q https://github.com/util-linux/util-linux/archive/refs/tags/v2.39.3.tar.gz tar -xf *.tar.gz +rm *.tar.gz cd util-linux* ./autogen.sh ./configure --host=${ARCH}-linux --disable-all-programs --enable-libuuid --prefix=${DEPSDIR} From ceef8c99a12a9df03a07c55037707109a75bd294 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 13:15:02 -0500 Subject: [PATCH 104/161] use gha artifact v4 --- .github/workflows/build_python.yml | 22 +++++++++++----------- .github/workflows/build_python_linux.yml | 16 ++++++++-------- .github/workflows/release_python.yml | 5 +++-- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 3080598f..5a833c73 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -57,16 +57,16 @@ jobs: ./scripts/build_windows.sh x86_64 ${{ inputs.python_version }} - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: python + name: python-windows-x86_64-${{ inputs.python_version }} path: ./python*.zip - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: - name: build-python + name: build-python-windows-x86_64-${{ inputs.python_version }} path: ./*python*.tar.gz test_windows: @@ -76,9 +76,9 @@ jobs: steps: - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: python + name: python-windows-x86_64-${{ inputs.python_version }} path: ./python/ - name: Test python in clean environment @@ -110,14 +110,14 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v3 with: - name: python + name: python-darwin-universal2-${{ inputs.python_version }} path: ./python*.zip - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: - name: build-python + name: build-python-darwin-universal2-${{ inputs.python_version }} path: ./*python*.tar.gz test_macos: @@ -127,9 +127,9 @@ jobs: steps: - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: python + name: python-darwin-universal2-${{ inputs.python_version }} path: ./python/ - name: Test python in clean environment diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index 6384e8eb..abdcc20d 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -42,16 +42,16 @@ jobs: ./scripts/build_linux_zig.sh ${{ inputs.arch }} ${{ inputs.python_version }} - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: python + name: python-linux-${{ inputs.arch }}-${{ inputs.python_version }} path: ./python*.zip - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: - name: build-python + name: build-python-linux-${{ inputs.arch }}-${{ inputs.python_version }} path: ./*python*.tar.gz - name: Test python in clean environment @@ -120,16 +120,16 @@ jobs: /work/scripts/build_linux.sh ${{ inputs.arch }} ${{ inputs.python_version }} - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: python + name: python-linux-${{ inputs.arch }}-${{ inputs.python_version }} path: ./python*.zip - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: - name: build-python + name: build-python-linux-${{ inputs.arch }}-${{ inputs.python_version }} path: ./*python*.tar.gz - name: Test python in clean environment diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 02284968..ed0c9691 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -41,10 +41,11 @@ jobs: run: git fetch --tags origin - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: python + pattern: python-* path: /tmp/python/ + merge-multiple: true - name: Pick tag run: | From 8ab0d7192b96e6333c527ba7885074e9f258f18a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 13:20:05 -0500 Subject: [PATCH 105/161] add libpng16 --- scripts/build_linux_zig.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 343290cb..2aacb18b 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -260,6 +260,21 @@ CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${AR make -j4 make install +echo "::endgroup::" +############ +# libpng16 # +############ +echo "::group::libpng16" +cd ${BUILDDIR} + +wget -q http://prdownloads.sourceforge.net/libpng/libpng-1.6.41.tar.gz +tar -xf libpng*.tar.gz +rm *.tar.gz +cd libpng* +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +make -j4 +make install + echo "::endgroup::" ############# # libgcrypt # From 027ad71a1f1dd3a2b0dd365ea682816ca6aaabfa Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 13:45:00 -0500 Subject: [PATCH 106/161] set cflags ldflags early --- scripts/build_linux_zig.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 2aacb18b..0f9e89db 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -57,6 +57,9 @@ export AR="${ARCH}-linux-gnu-gcc-ar" export CC="${ARCH}-linux-gnu-gcc" export CXX="${ARCH}-linux-gnu-g++" export CHOST=${ARCH} +export CFLAGS="-I${DEPSDIR}/include" +export LDFLAGS="-L${DEPSDIR}/lib" +export PKG_CONFIG_PATH="${DEPSDIR}/lib/pkgconfig" echo "::endgroup::" ######## @@ -256,7 +259,7 @@ wget -q https://download.gnome.org/sources/libxml2/2.12/libxml2-2.12.4.tar.xz tar -xf libxml2*.tar.xz rm *.tar.xz cd libxml2* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --without-python --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --without-python --prefix=${DEPSDIR} make -j4 make install @@ -296,7 +299,7 @@ wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.10.3.tar.bz2 tar -xf libgcrypt*.tar.bz2 rm *.tar.bz2 cd libgcrypt* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib -Wl,--undefined-version" ./configure --disable-asm --host=${ARCH}-linux --prefix=${DEPSDIR} +LDFLAGS="${LDFLAGS} -Wl,--undefined-version" ./configure --disable-asm --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -311,7 +314,7 @@ wget -q https://download.gnome.org/sources/libxslt/1.1/libxslt-1.1.39.tar.xz tar -xf libxslt*.tar.xz rm *.tar.xz cd libxslt* -CFLAGS="-I${DEPSDIR}/include -I${DEPSDIR}/include/libxml2" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --with-libxml-prefix=${DEPSDIR} --without-python --prefix=${DEPSDIR} +CFLAGS="${CFLAGS} -I${DEPSDIR}/include/libxml2" ./configure --host=${ARCH}-linux --with-libxml-prefix=${DEPSDIR} --without-python --prefix=${DEPSDIR} make -j4 make install @@ -326,7 +329,7 @@ wget -q https://download.savannah.gnu.org/releases/freetype/freetype-2.13.2.tar. tar -xf freetype*.tar.gz rm *.tar.gz cd freetype* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -341,7 +344,7 @@ wget -q https://www.freedesktop.org/software/fontconfig/release/fontconfig-2.15. tar -xf fontconfig*.tar.gz rm *.tar.gz cd fontconfig* -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" PKG_CONFIG_PATH="${DEPSDIR}/lib/pkgconfig" ./configure --host=${ARCH}-linux --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} make -j4 make install @@ -372,7 +375,7 @@ wget -q https://www.x.org/releases/individual/proto/scrnsaverproto-1.2.2.tar.gz wget -q https://www.x.org/releases/individual/xcb/libxcb-1.16.tar.gz wget -q https://www.x.org/releases/individual/xcb/libpthread-stubs-0.5.tar.gz git clone git://anongit.freedesktop.org/git/xorg/util/modular util/modular -CFLAGS="-I${DEPSDIR}/include" LDFLAGS="-L${DEPSDIR}/lib" ./util/modular/build.sh --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} +./util/modular/build.sh --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} rm *.tar.gz echo "::endgroup::" @@ -401,7 +404,7 @@ wget -q https://prdownloads.sourceforge.net/tcl/tk8.6.13-src.tar.gz tar -xf tk*.tar.gz rm *.tar.gz cd tk*/unix -CFLAGS="-I${DEPSDIR}/include" ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 774794dd2f1db43e1263b6c4440b217a66c871db Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 14:10:35 -0500 Subject: [PATCH 107/161] fix artifact upload --- .github/workflows/build_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 5a833c73..d1b05295 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -108,7 +108,7 @@ jobs: ./scripts/build_macos.sh universal2 ${{ inputs.python_version }} - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-darwin-universal2-${{ inputs.python_version }} path: ./python*.zip From 35df9d42b383a3fcba77120389e992580e1ec000 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 14:10:42 -0500 Subject: [PATCH 108/161] cross compile zlib --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 0f9e89db..6c46ff81 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -72,7 +72,7 @@ wget -q https://zlib.net/fossils/zlib-1.3.tar.gz tar -xf zlib*.tar.gz rm *.tar.gz cd zlib* -./configure --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 8ffe6126ab4167b7e48a19891b38a68d5e332c1b Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 15:02:43 -0500 Subject: [PATCH 109/161] Revert "cross compile zlib" This reverts commit 35df9d42b383a3fcba77120389e992580e1ec000. --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 6c46ff81..0f9e89db 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -72,7 +72,7 @@ wget -q https://zlib.net/fossils/zlib-1.3.tar.gz tar -xf zlib*.tar.gz rm *.tar.gz cd zlib* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --prefix=${DEPSDIR} make -j4 make install From 584c1f16287e4044358a7c8e9b3ac6e9589eb446 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 15:19:51 -0500 Subject: [PATCH 110/161] try older ncurses --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 0f9e89db..f7079bc6 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -158,7 +158,7 @@ echo "::endgroup::" echo "::group::ncurses" cd ${BUILDDIR} -wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz +wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.2.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* From 12e94911db1a04236abac8328e4436946c129d49 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 15:23:21 -0500 Subject: [PATCH 111/161] use explicit zlib prefix --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index f7079bc6..03953020 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -274,7 +274,7 @@ wget -q http://prdownloads.sourceforge.net/libpng/libpng-1.6.41.tar.gz tar -xf libpng*.tar.gz rm *.tar.gz cd libpng* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-zlib-prefix=${DEPSDIR} --prefix=${DEPSDIR} make -j4 make install From 58a27d055f110a6c9a64bea038de3c358b6c391b Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 15:46:58 -0500 Subject: [PATCH 112/161] disable libpng tools --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 03953020..bf43815a 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -274,7 +274,7 @@ wget -q http://prdownloads.sourceforge.net/libpng/libpng-1.6.41.tar.gz tar -xf libpng*.tar.gz rm *.tar.gz cd libpng* -./configure --host=${ARCH}-linux --with-zlib-prefix=${DEPSDIR} --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-zlib-prefix=${DEPSDIR} --disable-tools --prefix=${DEPSDIR} make -j4 make install From ed6adb2b754bdc6c1942e56fe0fb5b8e0bf03f93 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 16:04:57 -0500 Subject: [PATCH 113/161] try more fixes --- scripts/build_linux_zig.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index bf43815a..03026929 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -158,7 +158,7 @@ echo "::endgroup::" echo "::group::ncurses" cd ${BUILDDIR} -wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.2.tar.gz +wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* @@ -274,7 +274,7 @@ wget -q http://prdownloads.sourceforge.net/libpng/libpng-1.6.41.tar.gz tar -xf libpng*.tar.gz rm *.tar.gz cd libpng* -./configure --host=${ARCH}-linux --with-zlib-prefix=${DEPSDIR} --disable-tools --prefix=${DEPSDIR} +PNG_COPTS="${CFLAGS}" ./configure --host=${ARCH}-linux --with-zlib-prefix=${DEPSDIR} --disable-tools --prefix=${DEPSDIR} make -j4 make install From b4c31672a0152322a2995f56b9059b1ef04a0ce0 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 16:12:14 -0500 Subject: [PATCH 114/161] skip cross compile flag for ncurses on x86_64 --- scripts/build_linux_zig.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 03026929..db754e37 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -162,7 +162,11 @@ wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* -./configure --host=${ARCH}-linux --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +if [[ "${ARCH}" == "x86_64" ]]; then + ./configure --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +else + ./configure --host=${ARCH}-linux --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +fi make -j4 make install From 168e1a123ee859ba68e62cbd2534ab267248c9e6 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 16:14:14 -0500 Subject: [PATCH 115/161] add zlib dev headers --- scripts/build_linux_zig.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index db754e37..6f28336e 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -23,7 +23,7 @@ sudo apt update sudo apt -y install \ wget build-essential pkg-config cmake autoconf git \ python2 python3 python3-pip clang patchelf qemu-user-static \ - gettext bison libtool autopoint gperf + gettext bison libtool autopoint gperf zlib1g-dev case "$ARCH" in x86_64) sudo apt -y install libc6-amd64-cross @@ -278,7 +278,7 @@ wget -q http://prdownloads.sourceforge.net/libpng/libpng-1.6.41.tar.gz tar -xf libpng*.tar.gz rm *.tar.gz cd libpng* -PNG_COPTS="${CFLAGS}" ./configure --host=${ARCH}-linux --with-zlib-prefix=${DEPSDIR} --disable-tools --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-zlib-prefix=${DEPSDIR} --disable-tools --prefix=${DEPSDIR} make -j4 make install From c2c91a72b317882323cfea63e6010bc9baf96e18 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 16:25:42 -0500 Subject: [PATCH 116/161] Tar entire build dir --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 6f28336e..5849288c 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -11,7 +11,7 @@ WORKDIR=$(pwd) BUILDDIR=${WORKDIR}/build DEPSDIR=${WORKDIR}/deps -trap "cd ${BUILDDIR}/python-build && tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz ." EXIT +trap "cd ${BUILDDIR} && tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz ." EXIT ######################## # Install dependencies # From 0c5da2453dce0e954c414efb262f080c64e504a6 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 16:57:49 -0500 Subject: [PATCH 117/161] set cpp cxx flags --- scripts/build_linux_zig.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 5849288c..e486e5de 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -58,6 +58,8 @@ export CC="${ARCH}-linux-gnu-gcc" export CXX="${ARCH}-linux-gnu-g++" export CHOST=${ARCH} export CFLAGS="-I${DEPSDIR}/include" +export CPPFLAGS="-I${DEPSDIR}/include" +export CXXFLAGS="${CPPFLAGS}" export LDFLAGS="-L${DEPSDIR}/lib" export PKG_CONFIG_PATH="${DEPSDIR}/lib/pkgconfig" From 8daad71e82c061e11d56ad99e74d5ec4b52ff0a2 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 17:06:04 -0500 Subject: [PATCH 118/161] system ncurses binaries --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index e486e5de..408a8a13 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -23,7 +23,7 @@ sudo apt update sudo apt -y install \ wget build-essential pkg-config cmake autoconf git \ python2 python3 python3-pip clang patchelf qemu-user-static \ - gettext bison libtool autopoint gperf zlib1g-dev + gettext bison libtool autopoint gperf ncurses-bin case "$ARCH" in x86_64) sudo apt -y install libc6-amd64-cross From d3f7f943a92c968630fbe7a7d0e5b90c56a602dc Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 17:10:14 -0500 Subject: [PATCH 119/161] cross compile flags --- scripts/build_linux_zig.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 408a8a13..f39cce72 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -164,11 +164,7 @@ wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* -if [[ "${ARCH}" == "x86_64" ]]; then - ./configure --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} -else - ./configure --host=${ARCH}-linux --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} -fi +./configure --host=${ARCH}-linux --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install @@ -381,7 +377,7 @@ wget -q https://www.x.org/releases/individual/proto/scrnsaverproto-1.2.2.tar.gz wget -q https://www.x.org/releases/individual/xcb/libxcb-1.16.tar.gz wget -q https://www.x.org/releases/individual/xcb/libpthread-stubs-0.5.tar.gz git clone git://anongit.freedesktop.org/git/xorg/util/modular util/modular -./util/modular/build.sh --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} +./util/modular/build.sh --host ${ARCH}-linux --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} rm *.tar.gz echo "::endgroup::" From 116f6335f778606f9cb9f0eccf21873f611890e3 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 17:22:36 -0500 Subject: [PATCH 120/161] disable tic build --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index f39cce72..52370e61 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -164,7 +164,7 @@ wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz tar -xf ncurses*.tar.gz rm *.tar.gz cd ncurses* -./configure --host=${ARCH}-linux --with-normal --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-normal --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install From ad802498e64b8e8f29f9065eb51cf7fc2bc99b12 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 18:23:48 -0500 Subject: [PATCH 121/161] X11 compile flags --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 52370e61..d9750c3b 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -377,7 +377,7 @@ wget -q https://www.x.org/releases/individual/proto/scrnsaverproto-1.2.2.tar.gz wget -q https://www.x.org/releases/individual/xcb/libxcb-1.16.tar.gz wget -q https://www.x.org/releases/individual/xcb/libpthread-stubs-0.5.tar.gz git clone git://anongit.freedesktop.org/git/xorg/util/modular util/modular -./util/modular/build.sh --host ${ARCH}-linux --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} +./util/modular/build.sh --host ${ARCH}-unknown-linux-gnu --build x86_84-unknown-linux-gnu --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} rm *.tar.gz echo "::endgroup::" From d5d30b45156ba3d7f0a5a35413251615c5f4327f Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 18:26:26 -0500 Subject: [PATCH 122/161] Move configure flags to env var --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index d9750c3b..e7d247b6 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -377,7 +377,7 @@ wget -q https://www.x.org/releases/individual/proto/scrnsaverproto-1.2.2.tar.gz wget -q https://www.x.org/releases/individual/xcb/libxcb-1.16.tar.gz wget -q https://www.x.org/releases/individual/xcb/libpthread-stubs-0.5.tar.gz git clone git://anongit.freedesktop.org/git/xorg/util/modular util/modular -./util/modular/build.sh --host ${ARCH}-unknown-linux-gnu --build x86_84-unknown-linux-gnu --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} +CONFFLAGS="--host=${ARCH}-linux" ./util/modular/build.sh --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} rm *.tar.gz echo "::endgroup::" From 800fa0a2ee7a266c816d1a463668eb596105a0f4 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 18:37:59 -0500 Subject: [PATCH 123/161] Add gnu --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index e7d247b6..055dfc87 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -377,7 +377,7 @@ wget -q https://www.x.org/releases/individual/proto/scrnsaverproto-1.2.2.tar.gz wget -q https://www.x.org/releases/individual/xcb/libxcb-1.16.tar.gz wget -q https://www.x.org/releases/individual/xcb/libpthread-stubs-0.5.tar.gz git clone git://anongit.freedesktop.org/git/xorg/util/modular util/modular -CONFFLAGS="--host=${ARCH}-linux" ./util/modular/build.sh --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} +CONFFLAGS="--host=${ARCH}-linux-gnu" ./util/modular/build.sh --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} rm *.tar.gz echo "::endgroup::" From c9fcfbc26a86a93b2ba8b8f8a2de6cbefde70593 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 22:54:38 -0500 Subject: [PATCH 124/161] switch to cached dependency archives + add sha256sums --- checksums/brotli-1.1.0.tar.gz.sha256 | 1 + checksums/expat-2.5.0.tar.gz.sha256 | 1 + checksums/fontconfig-2.15.0.tar.gz.sha256 | 1 + checksums/freetype-2.13.2.tar.gz.sha256 | 1 + checksums/gdbm-1.23.tar.gz.sha256 | 1 + checksums/inputproto-2.3.2.tar.gz.sha256 | 1 + checksums/kbproto-1.0.7.tar.gz.sha256 | 1 + checksums/libICE-1.0.7.tar.gz.sha256 | 1 + checksums/libSM-1.2.2.tar.gz.sha256 | 1 + checksums/libX11-1.8.7.tar.gz.sha256 | 1 + checksums/libXScrnSaver-1.2.4.tar.gz.sha256 | 1 + checksums/libXau-1.0.11.tar.gz.sha256 | 1 + checksums/libXdmcp-1.1.2.tar.gz.sha256 | 1 + checksums/libXext-1.3.5.tar.gz.sha256 | 1 + checksums/libXft-2.3.8.tar.gz.sha256 | 1 + checksums/libXrender-0.9.11.tar.gz.sha256 | 1 + checksums/libffi-3.4.2.tar.gz.sha256 | 1 + checksums/libgcrypt-1.10.3.tar.bz2.sha256 | 1 + checksums/libgpg-error-1.47.tar.bz2.sha256 | 1 + checksums/libpng-1.6.41.tar.gz.sha256 | 1 + checksums/libpthread-stubs-0.5.tar.gz.sha256 | 1 + checksums/libxcb-1.16.tar.gz.sha256 | 1 + checksums/libxml2-2.12.4.tar.xz.sha256 | 1 + checksums/libxslt-1.1.39.tar.xz.sha256 | 1 + checksums/mpdecimal-2.5.0.tar.gz.sha256 | 1 + checksums/ncurses-6.4.tar.gz.sha256 | 1 + checksums/openssl-1.1.1w.tar.gz.sha256 | 1 + checksums/readline-8.2.tar.gz.sha256 | 1 + checksums/renderproto-0.11.1.tar.gz.sha256 | 1 + checksums/scrnsaverproto-1.2.2.tar.gz.sha256 | 1 + .../sqlite-autoconf-3450000.tar.gz.sha256 | 1 + checksums/tcl8.6.13-src.tar.gz.sha256 | 1 + checksums/tk8.6.13-src.tar.gz.sha256 | 1 + checksums/util-linux-2.39.3.tar.gz.sha256 | 1 + checksums/xcb-proto-1.16.0.tar.gz.sha256 | 1 + checksums/xextproto-7.3.0.tar.gz.sha256 | 1 + checksums/xproto-7.0.31.tar.gz.sha256 | 1 + checksums/xtrans-1.5.0.tar.gz.sha256 | 1 + checksums/xz-5.4.5.tar.gz.sha256 | 1 + checksums/zlib-1.3.1.tar.gz.sha256 | 1 + scripts/build_linux_zig.sh | 148 ++++++++---------- 41 files changed, 103 insertions(+), 85 deletions(-) create mode 100644 checksums/brotli-1.1.0.tar.gz.sha256 create mode 100644 checksums/expat-2.5.0.tar.gz.sha256 create mode 100644 checksums/fontconfig-2.15.0.tar.gz.sha256 create mode 100644 checksums/freetype-2.13.2.tar.gz.sha256 create mode 100644 checksums/gdbm-1.23.tar.gz.sha256 create mode 100644 checksums/inputproto-2.3.2.tar.gz.sha256 create mode 100644 checksums/kbproto-1.0.7.tar.gz.sha256 create mode 100644 checksums/libICE-1.0.7.tar.gz.sha256 create mode 100644 checksums/libSM-1.2.2.tar.gz.sha256 create mode 100644 checksums/libX11-1.8.7.tar.gz.sha256 create mode 100644 checksums/libXScrnSaver-1.2.4.tar.gz.sha256 create mode 100644 checksums/libXau-1.0.11.tar.gz.sha256 create mode 100644 checksums/libXdmcp-1.1.2.tar.gz.sha256 create mode 100644 checksums/libXext-1.3.5.tar.gz.sha256 create mode 100644 checksums/libXft-2.3.8.tar.gz.sha256 create mode 100644 checksums/libXrender-0.9.11.tar.gz.sha256 create mode 100644 checksums/libffi-3.4.2.tar.gz.sha256 create mode 100644 checksums/libgcrypt-1.10.3.tar.bz2.sha256 create mode 100644 checksums/libgpg-error-1.47.tar.bz2.sha256 create mode 100644 checksums/libpng-1.6.41.tar.gz.sha256 create mode 100644 checksums/libpthread-stubs-0.5.tar.gz.sha256 create mode 100644 checksums/libxcb-1.16.tar.gz.sha256 create mode 100644 checksums/libxml2-2.12.4.tar.xz.sha256 create mode 100644 checksums/libxslt-1.1.39.tar.xz.sha256 create mode 100644 checksums/mpdecimal-2.5.0.tar.gz.sha256 create mode 100644 checksums/ncurses-6.4.tar.gz.sha256 create mode 100644 checksums/openssl-1.1.1w.tar.gz.sha256 create mode 100644 checksums/readline-8.2.tar.gz.sha256 create mode 100644 checksums/renderproto-0.11.1.tar.gz.sha256 create mode 100644 checksums/scrnsaverproto-1.2.2.tar.gz.sha256 create mode 100644 checksums/sqlite-autoconf-3450000.tar.gz.sha256 create mode 100644 checksums/tcl8.6.13-src.tar.gz.sha256 create mode 100644 checksums/tk8.6.13-src.tar.gz.sha256 create mode 100644 checksums/util-linux-2.39.3.tar.gz.sha256 create mode 100644 checksums/xcb-proto-1.16.0.tar.gz.sha256 create mode 100644 checksums/xextproto-7.3.0.tar.gz.sha256 create mode 100644 checksums/xproto-7.0.31.tar.gz.sha256 create mode 100644 checksums/xtrans-1.5.0.tar.gz.sha256 create mode 100644 checksums/xz-5.4.5.tar.gz.sha256 create mode 100644 checksums/zlib-1.3.1.tar.gz.sha256 diff --git a/checksums/brotli-1.1.0.tar.gz.sha256 b/checksums/brotli-1.1.0.tar.gz.sha256 new file mode 100644 index 00000000..288008d7 --- /dev/null +++ b/checksums/brotli-1.1.0.tar.gz.sha256 @@ -0,0 +1 @@ +e720a6ca29428b803f4ad165371771f5398faba397edf6778837a18599ea13ff brotli-1.1.0.tar.gz \ No newline at end of file diff --git a/checksums/expat-2.5.0.tar.gz.sha256 b/checksums/expat-2.5.0.tar.gz.sha256 new file mode 100644 index 00000000..42a02189 --- /dev/null +++ b/checksums/expat-2.5.0.tar.gz.sha256 @@ -0,0 +1 @@ +6b902ab103843592be5e99504f846ec109c1abb692e85347587f237a4ffa1033 expat-2.5.0.tar.gz \ No newline at end of file diff --git a/checksums/fontconfig-2.15.0.tar.gz.sha256 b/checksums/fontconfig-2.15.0.tar.gz.sha256 new file mode 100644 index 00000000..4b1cc37d --- /dev/null +++ b/checksums/fontconfig-2.15.0.tar.gz.sha256 @@ -0,0 +1 @@ +f5f359d6332861bd497570848fcb42520964a9e83d5e3abe397b6b6db9bcaaf4 fontconfig-2.15.0.tar.gz \ No newline at end of file diff --git a/checksums/freetype-2.13.2.tar.gz.sha256 b/checksums/freetype-2.13.2.tar.gz.sha256 new file mode 100644 index 00000000..736eb3e4 --- /dev/null +++ b/checksums/freetype-2.13.2.tar.gz.sha256 @@ -0,0 +1 @@ +1ac27e16c134a7f2ccea177faba19801131116fd682efc1f5737037c5db224b5 freetype-2.13.2.tar.gz \ No newline at end of file diff --git a/checksums/gdbm-1.23.tar.gz.sha256 b/checksums/gdbm-1.23.tar.gz.sha256 new file mode 100644 index 00000000..f2e521dd --- /dev/null +++ b/checksums/gdbm-1.23.tar.gz.sha256 @@ -0,0 +1 @@ +74b1081d21fff13ae4bd7c16e5d6e504a4c26f7cde1dca0d963a484174bbcacd gdbm-1.23.tar.gz \ No newline at end of file diff --git a/checksums/inputproto-2.3.2.tar.gz.sha256 b/checksums/inputproto-2.3.2.tar.gz.sha256 new file mode 100644 index 00000000..8c31c4cf --- /dev/null +++ b/checksums/inputproto-2.3.2.tar.gz.sha256 @@ -0,0 +1 @@ +10eaadd531f38f7c92ab59ef0708ca195caf3164a75c4ed99f0c04f2913f6ef3 inputproto-2.3.2.tar.gz diff --git a/checksums/kbproto-1.0.7.tar.gz.sha256 b/checksums/kbproto-1.0.7.tar.gz.sha256 new file mode 100644 index 00000000..9561d61d --- /dev/null +++ b/checksums/kbproto-1.0.7.tar.gz.sha256 @@ -0,0 +1 @@ +828cb275b91268b1a3ea950d5c0c5eb076c678fdf005d517411f89cc8c3bb416 kbproto-1.0.7.tar.gz diff --git a/checksums/libICE-1.0.7.tar.gz.sha256 b/checksums/libICE-1.0.7.tar.gz.sha256 new file mode 100644 index 00000000..e4925563 --- /dev/null +++ b/checksums/libICE-1.0.7.tar.gz.sha256 @@ -0,0 +1 @@ +8463d080da6f87d5c20b21fe694dfb81f97319a8617030ea4720e97d1bb4659c libICE-1.0.7.tar.gz diff --git a/checksums/libSM-1.2.2.tar.gz.sha256 b/checksums/libSM-1.2.2.tar.gz.sha256 new file mode 100644 index 00000000..62a6a25c --- /dev/null +++ b/checksums/libSM-1.2.2.tar.gz.sha256 @@ -0,0 +1 @@ +14bb7c669ce2b8ff712fbdbf48120e3742a77edcd5e025d6b3325ed30cf120f4 libSM-1.2.2.tar.gz diff --git a/checksums/libX11-1.8.7.tar.gz.sha256 b/checksums/libX11-1.8.7.tar.gz.sha256 new file mode 100644 index 00000000..499a02e3 --- /dev/null +++ b/checksums/libX11-1.8.7.tar.gz.sha256 @@ -0,0 +1 @@ +793ebebf569f12c864b77401798d38814b51790fce206e01a431e5feb982e20b libX11-1.8.7.tar.gz \ No newline at end of file diff --git a/checksums/libXScrnSaver-1.2.4.tar.gz.sha256 b/checksums/libXScrnSaver-1.2.4.tar.gz.sha256 new file mode 100644 index 00000000..b7f6d69f --- /dev/null +++ b/checksums/libXScrnSaver-1.2.4.tar.gz.sha256 @@ -0,0 +1 @@ +0656b2630475104d6df75d91ebb8e0153e61d14e9871ef1f403bcda4a62a838a libXScrnSaver-1.2.4.tar.gz diff --git a/checksums/libXau-1.0.11.tar.gz.sha256 b/checksums/libXau-1.0.11.tar.gz.sha256 new file mode 100644 index 00000000..b8a966f2 --- /dev/null +++ b/checksums/libXau-1.0.11.tar.gz.sha256 @@ -0,0 +1 @@ +3a321aaceb803577a4776a5efe78836eb095a9e44bbc7a465d29463e1a14f189 libXau-1.0.11.tar.gz \ No newline at end of file diff --git a/checksums/libXdmcp-1.1.2.tar.gz.sha256 b/checksums/libXdmcp-1.1.2.tar.gz.sha256 new file mode 100644 index 00000000..fa1a8eed --- /dev/null +++ b/checksums/libXdmcp-1.1.2.tar.gz.sha256 @@ -0,0 +1 @@ +6f7c7e491a23035a26284d247779174dedc67e34e93cc3548b648ffdb6fc57c0 libXdmcp-1.1.2.tar.gz \ No newline at end of file diff --git a/checksums/libXext-1.3.5.tar.gz.sha256 b/checksums/libXext-1.3.5.tar.gz.sha256 new file mode 100644 index 00000000..d4e5ef45 --- /dev/null +++ b/checksums/libXext-1.3.5.tar.gz.sha256 @@ -0,0 +1 @@ +1a3dcda154f803be0285b46c9338515804b874b5ccc7a2b769ab7fd76f1035bd libXext-1.3.5.tar.gz diff --git a/checksums/libXft-2.3.8.tar.gz.sha256 b/checksums/libXft-2.3.8.tar.gz.sha256 new file mode 100644 index 00000000..52d4f7bd --- /dev/null +++ b/checksums/libXft-2.3.8.tar.gz.sha256 @@ -0,0 +1 @@ +32e48fe2d844422e64809e4e99b9d8aed26c1b541a5acf837c5037b8d9f278a8 libXft-2.3.8.tar.gz diff --git a/checksums/libXrender-0.9.11.tar.gz.sha256 b/checksums/libXrender-0.9.11.tar.gz.sha256 new file mode 100644 index 00000000..b7b71d2f --- /dev/null +++ b/checksums/libXrender-0.9.11.tar.gz.sha256 @@ -0,0 +1 @@ +6aec3ca02e4273a8cbabf811ff22106f641438eb194a12c0ae93c7e08474b667 libXrender-0.9.11.tar.gz diff --git a/checksums/libffi-3.4.2.tar.gz.sha256 b/checksums/libffi-3.4.2.tar.gz.sha256 new file mode 100644 index 00000000..aec8a5af --- /dev/null +++ b/checksums/libffi-3.4.2.tar.gz.sha256 @@ -0,0 +1 @@ +540fb721619a6aba3bdeef7d940d8e9e0e6d2c193595bc243241b77ff9e93620 libffi-3.4.2.tar.gz \ No newline at end of file diff --git a/checksums/libgcrypt-1.10.3.tar.bz2.sha256 b/checksums/libgcrypt-1.10.3.tar.bz2.sha256 new file mode 100644 index 00000000..6d0bbc23 --- /dev/null +++ b/checksums/libgcrypt-1.10.3.tar.bz2.sha256 @@ -0,0 +1 @@ +8b0870897ac5ac67ded568dcfadf45969cfa8a6beb0fd60af2a9eadc2a3272aa libgcrypt-1.10.3.tar.bz2 \ No newline at end of file diff --git a/checksums/libgpg-error-1.47.tar.bz2.sha256 b/checksums/libgpg-error-1.47.tar.bz2.sha256 new file mode 100644 index 00000000..ce88377c --- /dev/null +++ b/checksums/libgpg-error-1.47.tar.bz2.sha256 @@ -0,0 +1 @@ +9e3c670966b96ecc746c28c2c419541e3bcb787d1a73930f5e5f5e1bcbbb9bdb libgpg-error-1.47.tar.bz2 \ No newline at end of file diff --git a/checksums/libpng-1.6.41.tar.gz.sha256 b/checksums/libpng-1.6.41.tar.gz.sha256 new file mode 100644 index 00000000..0e3c8f25 --- /dev/null +++ b/checksums/libpng-1.6.41.tar.gz.sha256 @@ -0,0 +1 @@ +f00a11840f60616bdced9056d0f4cf2e4897697db039f15ce911704f957d3c5d libpng-1.6.41.tar.gz \ No newline at end of file diff --git a/checksums/libpthread-stubs-0.5.tar.gz.sha256 b/checksums/libpthread-stubs-0.5.tar.gz.sha256 new file mode 100644 index 00000000..7c5ba435 --- /dev/null +++ b/checksums/libpthread-stubs-0.5.tar.gz.sha256 @@ -0,0 +1 @@ +593196cc746173d1e25cb54a93a87fd749952df68699aab7e02c085530e87747 libpthread-stubs-0.5.tar.gz diff --git a/checksums/libxcb-1.16.tar.gz.sha256 b/checksums/libxcb-1.16.tar.gz.sha256 new file mode 100644 index 00000000..fdfcfda0 --- /dev/null +++ b/checksums/libxcb-1.16.tar.gz.sha256 @@ -0,0 +1 @@ +bc0f75f84b28e6496a19a1d094d7e47def861a50cb7cce5b23b62eecdc2a4479 libxcb-1.16.tar.gz diff --git a/checksums/libxml2-2.12.4.tar.xz.sha256 b/checksums/libxml2-2.12.4.tar.xz.sha256 new file mode 100644 index 00000000..e959ce52 --- /dev/null +++ b/checksums/libxml2-2.12.4.tar.xz.sha256 @@ -0,0 +1 @@ +497360e423cf0bd99eacdb7c6215dea92e6d6e89ee940393c2bae0e77cb9b7d0 libxml2-2.12.4.tar.xz \ No newline at end of file diff --git a/checksums/libxslt-1.1.39.tar.xz.sha256 b/checksums/libxslt-1.1.39.tar.xz.sha256 new file mode 100644 index 00000000..f6f75a45 --- /dev/null +++ b/checksums/libxslt-1.1.39.tar.xz.sha256 @@ -0,0 +1 @@ +2a20ad621148339b0759c4d4e96719362dee64c9a096dbba625ba053846349f0 libxslt-1.1.39.tar.xz \ No newline at end of file diff --git a/checksums/mpdecimal-2.5.0.tar.gz.sha256 b/checksums/mpdecimal-2.5.0.tar.gz.sha256 new file mode 100644 index 00000000..9169aa9a --- /dev/null +++ b/checksums/mpdecimal-2.5.0.tar.gz.sha256 @@ -0,0 +1 @@ +15417edc8e12a57d1d9d75fa7e3f22b158a3b98f44db9d694cfd2acde8dfa0ca mpdecimal-2.5.0.tar.gz diff --git a/checksums/ncurses-6.4.tar.gz.sha256 b/checksums/ncurses-6.4.tar.gz.sha256 new file mode 100644 index 00000000..19867692 --- /dev/null +++ b/checksums/ncurses-6.4.tar.gz.sha256 @@ -0,0 +1 @@ +6931283d9ac87c5073f30b6290c4c75f21632bb4fc3603ac8100812bed248159 ncurses-6.4.tar.gz \ No newline at end of file diff --git a/checksums/openssl-1.1.1w.tar.gz.sha256 b/checksums/openssl-1.1.1w.tar.gz.sha256 new file mode 100644 index 00000000..9b3e18b4 --- /dev/null +++ b/checksums/openssl-1.1.1w.tar.gz.sha256 @@ -0,0 +1 @@ + cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 openssl-1.1.1w.tar.gz \ No newline at end of file diff --git a/checksums/readline-8.2.tar.gz.sha256 b/checksums/readline-8.2.tar.gz.sha256 new file mode 100644 index 00000000..6176359c --- /dev/null +++ b/checksums/readline-8.2.tar.gz.sha256 @@ -0,0 +1 @@ +3feb7171f16a84ee82ca18a36d7b9be109a52c04f492a053331d7d1095007c35 readline-8.2.tar.gz \ No newline at end of file diff --git a/checksums/renderproto-0.11.1.tar.gz.sha256 b/checksums/renderproto-0.11.1.tar.gz.sha256 new file mode 100644 index 00000000..fb24b253 --- /dev/null +++ b/checksums/renderproto-0.11.1.tar.gz.sha256 @@ -0,0 +1 @@ +a0a4be3cad9381ae28279ba5582e679491fc2bec9aab8a65993108bf8dbce5fe renderproto-0.11.1.tar.gz diff --git a/checksums/scrnsaverproto-1.2.2.tar.gz.sha256 b/checksums/scrnsaverproto-1.2.2.tar.gz.sha256 new file mode 100644 index 00000000..81fd8e89 --- /dev/null +++ b/checksums/scrnsaverproto-1.2.2.tar.gz.sha256 @@ -0,0 +1 @@ +d8dee19c52977f65af08fad6aa237bacee11bc5a33e1b9b064e8ac1fd99d6e79 scrnsaverproto-1.2.2.tar.gz diff --git a/checksums/sqlite-autoconf-3450000.tar.gz.sha256 b/checksums/sqlite-autoconf-3450000.tar.gz.sha256 new file mode 100644 index 00000000..e0417f2b --- /dev/null +++ b/checksums/sqlite-autoconf-3450000.tar.gz.sha256 @@ -0,0 +1 @@ +72887d57a1d8f89f52be38ef84a6353ce8c3ed55ada7864eb944abd9a495e436 sqlite-autoconf-3450000.tar.gz \ No newline at end of file diff --git a/checksums/tcl8.6.13-src.tar.gz.sha256 b/checksums/tcl8.6.13-src.tar.gz.sha256 new file mode 100644 index 00000000..e652922b --- /dev/null +++ b/checksums/tcl8.6.13-src.tar.gz.sha256 @@ -0,0 +1 @@ +43a1fae7412f61ff11de2cfd05d28cfc3a73762f354a417c62370a54e2caf066 tcl8.6.13-src.tar.gz diff --git a/checksums/tk8.6.13-src.tar.gz.sha256 b/checksums/tk8.6.13-src.tar.gz.sha256 new file mode 100644 index 00000000..4f350979 --- /dev/null +++ b/checksums/tk8.6.13-src.tar.gz.sha256 @@ -0,0 +1 @@ +2e65fa069a23365440a3c56c556b8673b5e32a283800d8d9b257e3f584ce0675 tk8.6.13-src.tar.gz diff --git a/checksums/util-linux-2.39.3.tar.gz.sha256 b/checksums/util-linux-2.39.3.tar.gz.sha256 new file mode 100644 index 00000000..fd3e5e0b --- /dev/null +++ b/checksums/util-linux-2.39.3.tar.gz.sha256 @@ -0,0 +1 @@ +2434edd1cf2aaca2a2b76b5de5ce7c98b12f75af9f600800c0655af20be85956 util-linux-2.39.3.tar.gz \ No newline at end of file diff --git a/checksums/xcb-proto-1.16.0.tar.gz.sha256 b/checksums/xcb-proto-1.16.0.tar.gz.sha256 new file mode 100644 index 00000000..aa199be2 --- /dev/null +++ b/checksums/xcb-proto-1.16.0.tar.gz.sha256 @@ -0,0 +1 @@ +d9c7f010b1105fc3858bf07b5169b2dd8e7493c6652b1fe45f3321d874f291d7 xcb-proto-1.16.0.tar.gz diff --git a/checksums/xextproto-7.3.0.tar.gz.sha256 b/checksums/xextproto-7.3.0.tar.gz.sha256 new file mode 100644 index 00000000..b22620bb --- /dev/null +++ b/checksums/xextproto-7.3.0.tar.gz.sha256 @@ -0,0 +1 @@ +1b1bcdf91221e78c6c33738667a57bd9aaa63d5953174ad8ed9929296741c9f5 xextproto-7.3.0.tar.gz diff --git a/checksums/xproto-7.0.31.tar.gz.sha256 b/checksums/xproto-7.0.31.tar.gz.sha256 new file mode 100644 index 00000000..97a3323b --- /dev/null +++ b/checksums/xproto-7.0.31.tar.gz.sha256 @@ -0,0 +1 @@ +6d755eaae27b45c5cc75529a12855fed5de5969b367ed05003944cf901ed43c7 xproto-7.0.31.tar.gz diff --git a/checksums/xtrans-1.5.0.tar.gz.sha256 b/checksums/xtrans-1.5.0.tar.gz.sha256 new file mode 100644 index 00000000..5a5460a3 --- /dev/null +++ b/checksums/xtrans-1.5.0.tar.gz.sha256 @@ -0,0 +1 @@ +a806f8a92f879dcd0146f3f1153fdffe845f2fc0df9b1a26c19312b7b0a29c86 xtrans-1.5.0.tar.gz diff --git a/checksums/xz-5.4.5.tar.gz.sha256 b/checksums/xz-5.4.5.tar.gz.sha256 new file mode 100644 index 00000000..d0e480e0 --- /dev/null +++ b/checksums/xz-5.4.5.tar.gz.sha256 @@ -0,0 +1 @@ +135c90b934aee8fbc0d467de87a05cb70d627da36abe518c357a873709e5b7d6 xz-5.4.5.tar.gz \ No newline at end of file diff --git a/checksums/zlib-1.3.1.tar.gz.sha256 b/checksums/zlib-1.3.1.tar.gz.sha256 new file mode 100644 index 00000000..c8155b9f --- /dev/null +++ b/checksums/zlib-1.3.1.tar.gz.sha256 @@ -0,0 +1 @@ +9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 zlib-1.3.1.tar.gz \ No newline at end of file diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 52370e61..733fbd3e 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -11,6 +11,25 @@ WORKDIR=$(pwd) BUILDDIR=${WORKDIR}/build DEPSDIR=${WORKDIR}/deps +function verify_checksum () { + file=$1 + filename=$(basename $file) + sha256sum -c ${WORKDIR}/checksums/$file.sha256 +} + +function download_and_verify () { + file=$1 + wget --no-verbose https://github.com/bjia56/portable-python/releases/download/build-dependencies/$file + verify_checksum $file +} + +function download_verify_extract () { + file=$1 + download_and_verify $1 + tar -xf $file + rm $file +} + trap "cd ${BUILDDIR} && tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz ." EXIT ######################## @@ -70,9 +89,7 @@ echo "::endgroup::" echo "::group::zlib" cd ${BUILDDIR} -wget -q https://zlib.net/fossils/zlib-1.3.tar.gz -tar -xf zlib*.tar.gz -rm *.tar.gz +download_verify_extract zlib-1.3.1.tar.gz cd zlib* ./configure --prefix=${DEPSDIR} make -j4 @@ -85,10 +102,8 @@ echo "::endgroup::" echo "::group::OpenSSL" cd ${BUILDDIR} -wget -q https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar.gz -tar -xf OpenSSL*.tar.gz -rm *.tar.gz -cd openssl-OpenSSL* +download_verify_extract openssl-1.1.1w.tar.gz +cd openssl* ./Configure linux-${ARCH} no-shared --prefix=${DEPSDIR} --openssldir=${DEPSDIR} make -j4 make install_sw @@ -100,9 +115,7 @@ echo "::endgroup::" echo "::group::libffi" cd ${BUILDDIR} -wget -q https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz -tar -xf libffi*.tar.gz -rm *.tar.gz +download_verify_extract libffi-3.4.2.tar.gz cd libffi* ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -115,9 +128,7 @@ echo "::endgroup::" echo "::group::sqlite3" cd ${BUILDDIR} -wget -q https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz -tar -xf sqlite*.tar.gz -rm *.tar.gz +download_verify_extract sqlite-autoconf-3450000.tar.gz cd sqlite* ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -130,9 +141,7 @@ echo "::endgroup::" echo "::group::expat" cd ${BUILDDIR} -wget -q https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.gz -tar -xf expat*.tar.gz -rm *.tar.gz +download_verify_extract expat-2.5.0.tar.gz cd expat* ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -145,9 +154,7 @@ echo "::endgroup::" echo "::group::readline" cd ${BUILDDIR} -wget -q https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz -tar -xf readline*.tar.gz -rm *.tar.gz +download_verify_extract readline-8.2.tar.gz cd readline* ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -160,9 +167,7 @@ echo "::endgroup::" echo "::group::ncurses" cd ${BUILDDIR} -wget -q https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz -tar -xf ncurses*.tar.gz -rm *.tar.gz +download_verify_extract ncurses-6.4.tar.gz cd ncurses* ./configure --host=${ARCH}-linux --with-normal --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 @@ -175,7 +180,7 @@ echo "::endgroup::" echo "::group::bzip2" cd ${BUILDDIR} -wget -q -O bzip2.tar.gz https://github.com/commontk/bzip2/tarball/master +wget --no-verbose -O bzip2.tar.gz https://github.com/commontk/bzip2/tarball/master tar -xf bzip2*.tar.gz rm *.tar.gz cd commontk-bzip2* @@ -192,9 +197,7 @@ echo "::endgroup::" echo "::group::xz" cd ${BUILDDIR} -wget -q https://github.com/tukaani-project/xz/releases/download/v5.4.5/xz-5.4.5.tar.gz -tar -xf xz*.tar.gz -rm *.tar.gz +download_verify_extract xz-5.4.5.tar.gz cd xz* mkdir build cd build @@ -209,9 +212,7 @@ echo "::endgroup::" echo "::group::Brotli" cd ${BUILDDIR} -wget -q https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz -tar -xf *.tar.gz -rm *.tar.gz +download_verify_extract brotli-1.1.0.tar.gz cd brotli* mkdir build cd build @@ -226,9 +227,7 @@ echo "::endgroup::" echo "::group::uuid" cd ${BUILDDIR} -wget -q https://github.com/util-linux/util-linux/archive/refs/tags/v2.39.3.tar.gz -tar -xf *.tar.gz -rm *.tar.gz +download_verify_extract util-linux-2.39.3.tar.gz cd util-linux* ./autogen.sh ./configure --host=${ARCH}-linux --disable-all-programs --enable-libuuid --prefix=${DEPSDIR} @@ -242,9 +241,7 @@ echo "::endgroup::" echo "::group::gdbm" cd ${BUILDDIR} -wget -q https://ftp.gnu.org/gnu/gdbm/gdbm-1.23.tar.gz -tar -xf gdbm*.tar.gz -rm *.tar.gz +download_verify_extract gdbm-1.23.tar.gz cd gdbm* ./configure --host=${ARCH}-linux --enable-libgdbm-compat --prefix=${DEPSDIR} make -j4 @@ -257,9 +254,7 @@ echo "::endgroup::" echo "::group::libxml2" cd ${BUILDDIR} -wget -q https://download.gnome.org/sources/libxml2/2.12/libxml2-2.12.4.tar.xz -tar -xf libxml2*.tar.xz -rm *.tar.xz +download_verify_extract libxml2-2.12.4.tar.xz cd libxml2* ./configure --host=${ARCH}-linux --without-python --prefix=${DEPSDIR} make -j4 @@ -272,9 +267,7 @@ echo "::endgroup::" echo "::group::libpng16" cd ${BUILDDIR} -wget -q http://prdownloads.sourceforge.net/libpng/libpng-1.6.41.tar.gz -tar -xf libpng*.tar.gz -rm *.tar.gz +download_verify_extract libpng-1.6.41.tar.gz cd libpng* ./configure --host=${ARCH}-linux --with-zlib-prefix=${DEPSDIR} --disable-tools --prefix=${DEPSDIR} make -j4 @@ -287,9 +280,7 @@ echo "::endgroup::" echo "::group::libgcrypt" cd ${BUILDDIR} -wget -q https://gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.47.tar.bz2 -tar -xf libgpg-error*.tar.bz2 -rm *.tar.bz2 +download_verify_extract libgpg-error-1.47.tar.bz2 cd libgpg-error* ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -297,9 +288,7 @@ make install cd ${BUILDDIR} -wget -q https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.10.3.tar.bz2 -tar -xf libgcrypt*.tar.bz2 -rm *.tar.bz2 +download_verify_extract libgcrypt-1.10.3.tar.bz2 cd libgcrypt* LDFLAGS="${LDFLAGS} -Wl,--undefined-version" ./configure --disable-asm --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -312,9 +301,7 @@ echo "::endgroup::" echo "::group::libxslt" cd ${BUILDDIR} -wget -q https://download.gnome.org/sources/libxslt/1.1/libxslt-1.1.39.tar.xz -tar -xf libxslt*.tar.xz -rm *.tar.xz +download_verify_extract libxslt-1.1.39.tar.xz cd libxslt* CFLAGS="${CFLAGS} -I${DEPSDIR}/include/libxml2" ./configure --host=${ARCH}-linux --with-libxml-prefix=${DEPSDIR} --without-python --prefix=${DEPSDIR} make -j4 @@ -327,9 +314,7 @@ echo "::endgroup::" echo "::group::freetype" cd ${BUILDDIR} -wget -q https://download.savannah.gnu.org/releases/freetype/freetype-2.13.2.tar.gz -tar -xf freetype*.tar.gz -rm *.tar.gz +download_verify_extract freetype-2.13.2.tar.gz cd freetype* ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -342,9 +327,7 @@ echo "::endgroup::" echo "::group::fontconfig" cd ${BUILDDIR} -wget -q https://www.freedesktop.org/software/fontconfig/release/fontconfig-2.15.0.tar.gz -tar -xf fontconfig*.tar.gz -rm *.tar.gz +download_verify_extract fontconfig-2.15.0.tar.gz cd fontconfig* ./configure --host=${ARCH}-linux --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} make -j4 @@ -357,25 +340,25 @@ echo "::endgroup::" echo "::group::X11" cd ${BUILDDIR} -wget -q https://www.x.org/releases/individual/lib/libXau-1.0.11.tar.gz -wget -q https://www.x.org/releases/individual/lib/libXdmcp-1.1.2.tar.gz -wget -q https://www.x.org/releases/individual/lib/libX11-1.8.7.tar.gz -wget -q https://www.x.org/releases/individual/lib/libXext-1.3.5.tar.gz -wget -q https://www.x.org/releases/individual/lib/libICE-1.0.7.tar.gz -wget -q https://www.x.org/releases/individual/lib/libSM-1.2.2.tar.gz -wget -q https://www.x.org/releases/individual/lib/libXrender-0.9.11.tar.gz -wget -q https://www.x.org/releases/individual/lib/libXft-2.3.8.tar.gz -wget -q https://www.x.org/releases/individual/lib/libXScrnSaver-1.2.4.tar.gz -wget -q https://www.x.org/releases/individual/lib/xtrans-1.5.0.tar.gz -wget -q https://www.x.org/releases/individual/proto/xproto-7.0.31.tar.gz -wget -q https://www.x.org/releases/individual/proto/xextproto-7.3.0.tar.gz -wget -q https://www.x.org/releases/individual/proto/xcb-proto-1.16.0.tar.gz -wget -q https://www.x.org/releases/individual/proto/kbproto-1.0.7.tar.gz -wget -q https://www.x.org/releases/individual/proto/inputproto-2.3.2.tar.gz -wget -q https://www.x.org/releases/individual/proto/renderproto-0.11.1.tar.gz -wget -q https://www.x.org/releases/individual/proto/scrnsaverproto-1.2.2.tar.gz -wget -q https://www.x.org/releases/individual/xcb/libxcb-1.16.tar.gz -wget -q https://www.x.org/releases/individual/xcb/libpthread-stubs-0.5.tar.gz +download_and_verify libXau-1.0.11.tar.gz +download_and_verify libXdmcp-1.1.2.tar.gz +download_and_verify libX11-1.8.7.tar.gz +download_and_verify libXext-1.3.5.tar.gz +download_and_verify libICE-1.0.7.tar.gz +download_and_verify libSM-1.2.2.tar.gz +download_and_verify libXrender-0.9.11.tar.gz +download_and_verify libXft-2.3.8.tar.gz +download_and_verify libXScrnSaver-1.2.4.tar.gz +download_and_verify xtrans-1.5.0.tar.gz +download_and_verify xproto-7.0.31.tar.gz +download_and_verify xextproto-7.3.0.tar.gz +download_and_verify xcb-proto-1.16.0.tar.gz +download_and_verify kbproto-1.0.7.tar.gz +download_and_verify inputproto-2.3.2.tar.gz +download_and_verify renderproto-0.11.1.tar.gz +download_and_verify scrnsaverproto-1.2.2.tar.gz +download_and_verify libxcb-1.16.tar.gz +download_and_verify libpthread-stubs-0.5.tar.gz git clone git://anongit.freedesktop.org/git/xorg/util/modular util/modular ./util/modular/build.sh --host ${ARCH}-linux --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} rm *.tar.gz @@ -387,9 +370,7 @@ echo "::endgroup::" echo "::group::tcl" cd ${BUILDDIR} -wget -q https://prdownloads.sourceforge.net/tcl/tcl8.6.13-src.tar.gz -tar -xf tcl*.tar.gz -rm *.tar.gz +download_verify_extract tcl8.6.13-src.tar.gz cd tcl*/unix ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -402,9 +383,7 @@ echo "::endgroup::" echo "::group::tk" cd ${BUILDDIR} -wget -q https://prdownloads.sourceforge.net/tcl/tk8.6.13-src.tar.gz -tar -xf tk*.tar.gz -rm *.tar.gz +download_verify_extract tk8.6.13-src.tar.gz cd tk*/unix ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 @@ -417,9 +396,8 @@ echo "::endgroup::" echo "::group::mpdecimal" cd ${BUILDDIR} -wget -q https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.0.tar.gz -tar -xzf mpdecimal*.tar.gz -cd mpdecimal-2.5.0 +download_verify_extract mpdecimal-2.5.0.tar.gz +cd mpdecimal* ./configure --disable-cxx --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -437,7 +415,7 @@ else INSTALL_TEST="OFF" fi -wget -q -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/portable-python +wget --no-verbose -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/portable-python tar -xf python-cmake-buildsystem.tar.gz rm *.tar.gz mv *python-cmake-buildsystem* python-cmake-buildsystem From 64e0dfdec8f9d5a441da1da87f8354e96f48e119 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Mon, 29 Jan 2024 23:22:20 -0500 Subject: [PATCH 125/161] build ncurses shared --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index d487a7f1..e01f8253 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -169,7 +169,7 @@ cd ${BUILDDIR} download_verify_extract ncurses-6.4.tar.gz cd ncurses* -./configure --host=${ARCH}-linux --with-normal --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-normal --with-shared --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install From f7a307ffdbf28bb03d7e5a48fe8701ab6872bad7 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 00:03:34 -0500 Subject: [PATCH 126/161] Use libtool --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index e01f8253..13131a4e 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -169,7 +169,7 @@ cd ${BUILDDIR} download_verify_extract ncurses-6.4.tar.gz cd ncurses* -./configure --host=${ARCH}-linux --with-normal --with-shared --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-normal --with-libtool --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install From 024866527b3e408dbb3725eb33c0ad7ce0ee7bb7 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 00:27:01 -0500 Subject: [PATCH 127/161] Libtool path --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 13131a4e..bfa2afbf 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -169,7 +169,7 @@ cd ${BUILDDIR} download_verify_extract ncurses-6.4.tar.gz cd ncurses* -./configure --host=${ARCH}-linux --with-normal --with-libtool --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-normal --with-libtool=/usr/bin/libtool --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install From b76192e40d805311c2371717fdd41a031f296e88 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 06:36:04 -0500 Subject: [PATCH 128/161] Add libtool bin --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index bfa2afbf..8fd71748 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -42,7 +42,7 @@ sudo apt update sudo apt -y install \ wget build-essential pkg-config cmake autoconf git \ python2 python3 python3-pip clang patchelf qemu-user-static \ - gettext bison libtool autopoint gperf ncurses-bin + gettext bison libtool libtool-bin autopoint gperf ncurses-bin case "$ARCH" in x86_64) sudo apt -y install libc6-amd64-cross From 7a081398eded6d34167cf31762f9a04ff43848f9 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 06:50:13 -0500 Subject: [PATCH 129/161] link readline against curses --- scripts/build_linux_zig.sh | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 8fd71748..1d90115f 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -147,19 +147,6 @@ cd expat* make -j4 make install -echo "::endgroup::" -############ -# readline # -############ -echo "::group::readline" -cd ${BUILDDIR} - -download_verify_extract readline-8.2.tar.gz -cd readline* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} -make -j4 -make install - echo "::endgroup::" ########### # ncurses # @@ -173,7 +160,20 @@ cd ncurses* make -j4 make install -echo "::endgroup::" +echo "::endgroup::" +############ +# readline # +############ +echo "::group::readline" +cd ${BUILDDIR} + +download_verify_extract readline-8.2.tar.gz +cd readline* +./configure --with-curses --host=${ARCH}-linux --prefix=${DEPSDIR} +make -j4 +make install + +echo "::endgroup::" ######### # bzip2 # ######### From 94e13f481f86ae77245a3cb4a5aeafa37eb5dd6c Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 07:04:55 -0500 Subject: [PATCH 130/161] Disable shared readline --- scripts/build_linux_zig.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 1d90115f..06992067 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -42,7 +42,7 @@ sudo apt update sudo apt -y install \ wget build-essential pkg-config cmake autoconf git \ python2 python3 python3-pip clang patchelf qemu-user-static \ - gettext bison libtool libtool-bin autopoint gperf ncurses-bin + gettext bison libtool autopoint gperf ncurses-bin case "$ARCH" in x86_64) sudo apt -y install libc6-amd64-cross @@ -156,7 +156,7 @@ cd ${BUILDDIR} download_verify_extract ncurses-6.4.tar.gz cd ncurses* -./configure --host=${ARCH}-linux --with-normal --with-libtool=/usr/bin/libtool --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --with-normal --without-progs --enable-overwrite --disable-stripping --prefix=${DEPSDIR} make -j4 make install @@ -169,7 +169,7 @@ cd ${BUILDDIR} download_verify_extract readline-8.2.tar.gz cd readline* -./configure --with-curses --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --with-curses --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 24c74411bc41e3a1a0fe8828da1af30bce628556 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 09:20:08 -0500 Subject: [PATCH 131/161] build x11 libs individually --- scripts/build_linux_zig.sh | 62 +++++++++++++++++++++++--------------- scripts/x11_modfile.txt | 19 ------------ 2 files changed, 37 insertions(+), 44 deletions(-) delete mode 100644 scripts/x11_modfile.txt diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 06992067..1512d149 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -42,7 +42,7 @@ sudo apt update sudo apt -y install \ wget build-essential pkg-config cmake autoconf git \ python2 python3 python3-pip clang patchelf qemu-user-static \ - gettext bison libtool autopoint gperf ncurses-bin + gettext bison libtool autopoint gperf ncurses-bin xutils-dev case "$ARCH" in x86_64) sudo apt -y install libc6-amd64-cross @@ -160,7 +160,7 @@ cd ncurses* make -j4 make install -echo "::endgroup::" +echo "::endgroup::" ############ # readline # ############ @@ -173,7 +173,7 @@ cd readline* make -j4 make install -echo "::endgroup::" +echo "::endgroup::" ######### # bzip2 # ######### @@ -340,28 +340,40 @@ echo "::endgroup::" echo "::group::X11" cd ${BUILDDIR} -download_and_verify libXau-1.0.11.tar.gz -download_and_verify libXdmcp-1.1.2.tar.gz -download_and_verify libX11-1.8.7.tar.gz -download_and_verify libXext-1.3.5.tar.gz -download_and_verify libICE-1.0.7.tar.gz -download_and_verify libSM-1.2.2.tar.gz -download_and_verify libXrender-0.9.11.tar.gz -download_and_verify libXft-2.3.8.tar.gz -download_and_verify libXScrnSaver-1.2.4.tar.gz -download_and_verify xtrans-1.5.0.tar.gz -download_and_verify xproto-7.0.31.tar.gz -download_and_verify xextproto-7.3.0.tar.gz -download_and_verify xcb-proto-1.16.0.tar.gz -download_and_verify kbproto-1.0.7.tar.gz -download_and_verify inputproto-2.3.2.tar.gz -download_and_verify renderproto-0.11.1.tar.gz -download_and_verify scrnsaverproto-1.2.2.tar.gz -download_and_verify libxcb-1.16.tar.gz -download_and_verify libpthread-stubs-0.5.tar.gz -git clone git://anongit.freedesktop.org/git/xorg/util/modular util/modular -CONFFLAGS="--host=${ARCH}-linux-gnu" ./util/modular/build.sh --modfile ${WORKDIR}/scripts/x11_modfile.txt ${DEPSDIR} -rm *.tar.gz +function build_x11_lib () { + cd ${BUILDDIR} + + pkg=$1 + file=$pkg.tar.gz + download_and_verify $file + cd $pkg + autoreconf -vfi + ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} + make -j4 + make install + + cd ${BUILDDIR} +} + +build_x11_lib xproto-7.0.31 +build_x11_lib xextproto-7.3.0 +build_x11_lib kbproto-1.0.7 +build_x11_lib inputproto-2.3.2 +build_x11_lib renderproto-0.11.1 +build_x11_lib scrnsaverproto-1.2.2 +build_x11_lib xcb-proto-1.16.0 +build_x11_lib libpthread-stubs-0.5 +build_x11_lib xtrans-1.5.0 +build_x11_lib libXau-1.0.11 +build_x11_lib libxcb-1.16 +build_x11_lib libXdmcp-1.1.2 +build_x11_lib libX11-1.8.7 +build_x11_lib libXext-1.3.5 +build_x11_lib libICE-1.0.7 +build_x11_lib libSM-1.2.2 +build_x11_lib libXrender-0.9.11 +build_x11_lib libXft-2.3.8 +build_x11_lib libXScrnSaver-1.2.4 echo "::endgroup::" ####### diff --git a/scripts/x11_modfile.txt b/scripts/x11_modfile.txt deleted file mode 100644 index 207a5371..00000000 --- a/scripts/x11_modfile.txt +++ /dev/null @@ -1,19 +0,0 @@ -proto/xproto -proto/xextproto -proto/kbproto -proto/inputproto -proto/renderproto -proto/scrnsaverproto -xcb/proto -xcb/pthread-stubs -lib/xtrans -lib/libXau -lib/libxcb -lib/libXdmcp -lib/libX11 -lib/libXext -lib/libICE -lib/libSM -lib/libXrender -lib/libXft -lib/libXScrnSaver From 5ea93c5d65fb5070dc6bad1fe919ec27abe69cd1 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 09:20:58 -0500 Subject: [PATCH 132/161] use correct bash function --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 1512d149..df9790e9 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -345,7 +345,7 @@ function build_x11_lib () { pkg=$1 file=$pkg.tar.gz - download_and_verify $file + download_verify_extract $file cd $pkg autoreconf -vfi ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} From 802150369a9a0ba0a9a9ccf2fe6b509d0ca3b4af Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 09:43:20 -0500 Subject: [PATCH 133/161] some x11 libs install to share/pkgconfig --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index df9790e9..d204cc88 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -80,7 +80,7 @@ export CFLAGS="-I${DEPSDIR}/include" export CPPFLAGS="-I${DEPSDIR}/include" export CXXFLAGS="${CPPFLAGS}" export LDFLAGS="-L${DEPSDIR}/lib" -export PKG_CONFIG_PATH="${DEPSDIR}/lib/pkgconfig" +export PKG_CONFIG_PATH="${DEPSDIR}/lib/pkgconfig:${DEPSDIR}/share/pkgconfig" echo "::endgroup::" ######## From ad41689ce6939f9fcb4c199ab5678dc645f44a6d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 09:56:25 -0500 Subject: [PATCH 134/161] add xorgproto --- checksums/xorgproto-2023.2.tar.gz.sha256 | 1 + scripts/build_linux_zig.sh | 1 + 2 files changed, 2 insertions(+) create mode 100644 checksums/xorgproto-2023.2.tar.gz.sha256 diff --git a/checksums/xorgproto-2023.2.tar.gz.sha256 b/checksums/xorgproto-2023.2.tar.gz.sha256 new file mode 100644 index 00000000..71ff3b69 --- /dev/null +++ b/checksums/xorgproto-2023.2.tar.gz.sha256 @@ -0,0 +1 @@ +c791aad9b5847781175388ebe2de85cb5f024f8dabf526d5d699c4f942660cc3 xorgproto-2023.2.tar.gz diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index d204cc88..46d3e3da 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -355,6 +355,7 @@ function build_x11_lib () { cd ${BUILDDIR} } +build_x11_lib xorgproto-2023.2 build_x11_lib xproto-7.0.31 build_x11_lib xextproto-7.3.0 build_x11_lib kbproto-1.0.7 From 54e6b1c3ab1012b6dd6a98051b0f79baac230de1 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 10:30:06 -0500 Subject: [PATCH 135/161] tweak ci output --- scripts/build_linux_zig.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 46d3e3da..1aeba475 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -337,10 +337,11 @@ echo "::endgroup::" ####### # X11 # ####### -echo "::group::X11" -cd ${BUILDDIR} +#echo "::group::X11" +#cd ${BUILDDIR} function build_x11_lib () { + echo "::group::$1" cd ${BUILDDIR} pkg=$1 @@ -352,7 +353,7 @@ function build_x11_lib () { make -j4 make install - cd ${BUILDDIR} + echo "::endgroup::" } build_x11_lib xorgproto-2023.2 @@ -376,7 +377,7 @@ build_x11_lib libXrender-0.9.11 build_x11_lib libXft-2.3.8 build_x11_lib libXScrnSaver-1.2.4 -echo "::endgroup::" +#echo "::endgroup::" ####### # tcl # ####### @@ -435,7 +436,7 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -CFLAGS="-I${DEPSDIR}/include" cmake \ +cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_C_STANDARD=99 \ From df3e840a18a3ab7201b59d8ca0fe0903ea2d4e34 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 13:08:27 -0500 Subject: [PATCH 136/161] malloc(0) returns null --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 1aeba475..25fe4d27 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -349,7 +349,7 @@ function build_x11_lib () { download_verify_extract $file cd $pkg autoreconf -vfi - ./configure --host=${ARCH}-linux --prefix=${DEPSDIR} + ./configure --enable-malloc0returnsnull --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From fb30c5f25b9f56b5ab3bdf21005bcf920b1a6ffc Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 13:16:10 -0500 Subject: [PATCH 137/161] selectively set configure flags --- scripts/build_linux_zig.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 25fe4d27..7c9058d8 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -345,11 +345,12 @@ function build_x11_lib () { cd ${BUILDDIR} pkg=$1 + ext_flags="$2" file=$pkg.tar.gz download_verify_extract $file cd $pkg autoreconf -vfi - ./configure --enable-malloc0returnsnull --host=${ARCH}-linux --prefix=${DEPSDIR} + ./configure $ext_flags --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -369,13 +370,13 @@ build_x11_lib xtrans-1.5.0 build_x11_lib libXau-1.0.11 build_x11_lib libxcb-1.16 build_x11_lib libXdmcp-1.1.2 -build_x11_lib libX11-1.8.7 -build_x11_lib libXext-1.3.5 +build_x11_lib libX11-1.8.7 --enable-malloc0returnsnull +build_x11_lib libXext-1.3.5 --enable-malloc0returnsnull build_x11_lib libICE-1.0.7 build_x11_lib libSM-1.2.2 -build_x11_lib libXrender-0.9.11 +build_x11_lib libXrender-0.9.11 --enable-malloc0returnsnull build_x11_lib libXft-2.3.8 -build_x11_lib libXScrnSaver-1.2.4 +build_x11_lib libXScrnSaver-1.2.4 --enable-malloc0returnsnull #echo "::endgroup::" ####### From 5873abd78cf8c4857486a518f60aa9114d5c76e8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 13:45:27 -0500 Subject: [PATCH 138/161] reduce external source pulls --- .github/workflows/build_python.yml | 3 ++ .../sqlite-amalgamation-3430100.zip.sha256 | 1 + scripts/build_linux.sh | 12 +++---- scripts/build_linux_zig.sh | 31 ++--------------- scripts/build_macos.sh | 20 +++++------ scripts/build_windows.sh | 18 ++++------ scripts/utils.sh | 34 +++++++++++++++++++ 7 files changed, 61 insertions(+), 58 deletions(-) create mode 100644 checksums/sqlite-amalgamation-3430100.zip.sha256 create mode 100644 scripts/utils.sh diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index d1b05295..de29d50b 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -102,6 +102,9 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Install coreutils + run: brew install coreutils + - name: Build run: | set -ex diff --git a/checksums/sqlite-amalgamation-3430100.zip.sha256 b/checksums/sqlite-amalgamation-3430100.zip.sha256 new file mode 100644 index 00000000..457593ba --- /dev/null +++ b/checksums/sqlite-amalgamation-3430100.zip.sha256 @@ -0,0 +1 @@ +7e634bbd4b2870a83dc7c1e3cc02e4d30b8555cd7db7b332f24e0c447fd0dd16 sqlite-amalgamation-3430100.zip diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh index a02e1dd7..5a6fec13 100755 --- a/scripts/build_linux.sh +++ b/scripts/build_linux.sh @@ -1,10 +1,8 @@ #!/bin/bash -ARCH=$1 -PYTHON_FULL_VER=$2 -PYTHON_VER=$(echo ${PYTHON_FULL_VER} | cut -d "." -f 1-2) - -set -ex +PLATFORM=linux +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source ${SCRIPT_DIR}/utils.sh ######################## # Install dependencies # @@ -34,7 +32,7 @@ case "$ARCH" in wget -q https://files.pythonhosted.org/packages/0d/41/85549e9645097cddc7279886eafeafc7462215f309133ea2eae4941f9c35/cmake-3.26.4-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl ;; armv7l) - wget -q https://www.piwheels.org/simple/cmake/cmake-3.26.4-cp37-cp37m-linux_armv7l.whl + wget -q https://github.com/bjia56/armv7l-wheels/releases/download/cmake-3.26.4-cpython3.7/cmake-3.26.4-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl ;; esac mv cmake*.whl cmake-3.26.4-py2.py3-none-any.whl @@ -70,7 +68,7 @@ echo "::group::Run build" if [[ "${ARCH}" == "armv7l" ]]; then cd /tmp - wget -q https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.0.tar.gz + wget -q https://github.com/bjia56/portable-python/releases/download/build-dependencies/mpdecimal-2.5.0.tar.gz tar -xzf mpdecimal*.tar.gz cd mpdecimal-2.5.0 ./configure diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 7c9058d8..894f9170 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -1,36 +1,11 @@ #!/bin/bash -ARCH=$1 -PYTHON_FULL_VER=$2 -PYTHON_VER=$(echo ${PYTHON_FULL_VER} | cut -d "." -f 1-2) +PLATFORM=linux +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source ${SCRIPT_DIR}/utils.sh -set -ex zig version -WORKDIR=$(pwd) -BUILDDIR=${WORKDIR}/build -DEPSDIR=${WORKDIR}/deps - -function verify_checksum () { - file=$1 - filename=$(basename $file) - sha256sum -c ${WORKDIR}/checksums/$file.sha256 -} - -function download_and_verify () { - file=$1 - wget --no-verbose https://github.com/bjia56/portable-python/releases/download/build-dependencies/$file - verify_checksum $file -} - -function download_verify_extract () { - file=$1 - download_and_verify $1 - tar -xf $file - rm $file -} - -trap "cd ${BUILDDIR} && tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-linux-${ARCH}.tar.gz ." EXIT ######################## # Install dependencies # diff --git a/scripts/build_macos.sh b/scripts/build_macos.sh index 9e4093d5..44042e8e 100755 --- a/scripts/build_macos.sh +++ b/scripts/build_macos.sh @@ -1,10 +1,9 @@ #!/bin/bash -ARCH=$1 -PYTHON_FULL_VER=$2 -PYTHON_VER=$(echo ${PYTHON_FULL_VER} | cut -d "." -f 1-2) +PLATFORM=darwin +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source ${SCRIPT_DIR}/utils.sh -WORKDIR=$(pwd) NPROC=$(sysctl -n hw.ncpu) set -ex @@ -29,8 +28,7 @@ echo "::endgroup::" echo "::group::OpenSSL" cd ${WORKDIR} -wget -q https://www.openssl.org/source/openssl-1.1.1w.tar.gz -tar -xf openssl-1.1.1w.tar.gz +download_verify_extract openssl-1.1.1w.tar.gz mkdir deps/openssl cd openssl-1.1.1w @@ -99,10 +97,9 @@ echo "::endgroup::" echo "::group::sqlite3" cd ${WORKDIR} -wget -q https://www.sqlite.org/2023/sqlite-autoconf-3430100.tar.gz -tar -xf sqlite-autoconf-3430100.tar.gz +download_verify_extract sqlite-autoconf-3450000.tar.gz mkdir deps/sqlite3 -cd sqlite-autoconf-3430100 +cd sqlite-autoconf-3450000 CC=clang CFLAGS="-arch x86_64 -arch arm64" ./configure --prefix ${WORKDIR}/deps/sqlite3 make -j${NPROC} make install @@ -116,10 +113,9 @@ echo "::endgroup::" echo "::group::zlib" cd ${WORKDIR} -curl -L https://zlib.net/fossils/zlib-1.3.tar.gz --output zlib.tar.gz -tar -xf zlib.tar.gz +download_verify_extract zlib-1.3.1.tar.gz mkdir deps/zlib -cd zlib-1.3 +cd zlib-1.3.1 mkdir build cd build cmake \ diff --git a/scripts/build_windows.sh b/scripts/build_windows.sh index d13fca4d..eba2d3f5 100755 --- a/scripts/build_windows.sh +++ b/scripts/build_windows.sh @@ -1,11 +1,8 @@ #!/bin/bash -ARCH=$1 -PYTHON_FULL_VER=$2 - -WORKDIR=$(pwd) - -set -ex +PLATFORM=windows +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source ${SCRIPT_DIR}/utils.sh ############## # Initialize # @@ -74,8 +71,8 @@ echo "::endgroup::" echo "::group::sqlite3" cd ${WORKDIR} -curl -L https://www.sqlite.org/2023/sqlite-amalgamation-3430100.zip --output sqlite3-src.zip -unzip -qq sqlite3-src.zip +download_and_verify sqlite-amalgamation-3430100.zip +unzip -qq sqlite3-*.zip mv sqlite-amalgamation-3430100 deps/sqlite3 cd deps/sqlite3 cl //c sqlite3.c @@ -88,10 +85,9 @@ echo "::endgroup::" echo "::group::zlib" cd ${WORKDIR} -curl -L https://zlib.net/fossils/zlib-1.3.tar.gz --output zlib.tar.gz -tar -xf zlib.tar.gz +download_verify_extract https://zlib.net/fossils/zlib-1.3.1.tar.gz mkdir deps/zlib -cd zlib-1.3 +cd zlib-1.3.1 mkdir build cd build cmake \ diff --git a/scripts/utils.sh b/scripts/utils.sh new file mode 100644 index 00000000..94418590 --- /dev/null +++ b/scripts/utils.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" + +function verify_checksum () { + file=$1 + filename=$(basename $file) + sha256sum -c ${SCRIPT_DIR}/../checksums/$file.sha256 +} + +function download_and_verify () { + file=$1 + curl -s -o $file https://github.com/bjia56/portable-python/releases/download/build-dependencies/$file + verify_checksum $file +} + +function download_verify_extract () { + file=$1 + download_and_verify $1 + tar -xf $file + rm $file +} + +ARCH=$1 +PYTHON_FULL_VER=$2 +PYTHON_VER=$(echo ${PYTHON_FULL_VER} | cut -d "." -f 1-2) + +WORKDIR=$(pwd) +BUILDDIR=${WORKDIR}/build +DEPSDIR=${WORKDIR}/deps + +trap "cd ${BUILDDIR} && tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-${PLATFORM}-${ARCH}.tar.gz ." EXIT From 3b21c86f1cb407bd11a4a1c2d75e4ddc6d0e6dba Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 13:48:28 -0500 Subject: [PATCH 139/161] curl follow redirects --- scripts/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/utils.sh b/scripts/utils.sh index 94418590..a571b155 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -12,7 +12,7 @@ function verify_checksum () { function download_and_verify () { file=$1 - curl -s -o $file https://github.com/bjia56/portable-python/releases/download/build-dependencies/$file + curl -s -L -o $file https://github.com/bjia56/portable-python/releases/download/build-dependencies/$file verify_checksum $file } From e94e8fe20b6250c80d54a46ca69dd7a69119e2ef Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 13:53:18 -0500 Subject: [PATCH 140/161] minor tweaks --- scripts/build_windows.sh | 2 +- scripts/utils.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/build_windows.sh b/scripts/build_windows.sh index eba2d3f5..1c5f19f0 100755 --- a/scripts/build_windows.sh +++ b/scripts/build_windows.sh @@ -72,7 +72,7 @@ echo "::group::sqlite3" cd ${WORKDIR} download_and_verify sqlite-amalgamation-3430100.zip -unzip -qq sqlite3-*.zip +unzip -qq sqlite-amalgamation-3430100.zip mv sqlite-amalgamation-3430100 deps/sqlite3 cd deps/sqlite3 cl //c sqlite3.c diff --git a/scripts/utils.sh b/scripts/utils.sh index a571b155..98e2aa63 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -5,20 +5,20 @@ set -e SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" function verify_checksum () { - file=$1 + file="$1" filename=$(basename $file) sha256sum -c ${SCRIPT_DIR}/../checksums/$file.sha256 } function download_and_verify () { - file=$1 + file="$1" curl -s -L -o $file https://github.com/bjia56/portable-python/releases/download/build-dependencies/$file verify_checksum $file } function download_verify_extract () { - file=$1 - download_and_verify $1 + file="$1" + download_and_verify $file tar -xf $file rm $file } From 8a1df786a4a933053a00a7aefc9e82b0ff20ab4a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 14:08:06 -0500 Subject: [PATCH 141/161] fix windows lf --- .github/workflows/build_python.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index de29d50b..cd70e84f 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -39,6 +39,11 @@ jobs: runs-on: windows-latest steps: + - name: Set git to use LF + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: Checkout uses: actions/checkout@v3 From 8102962c7f88c0a5626c7ed0e31737c5a1f354eb Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 14:15:44 -0500 Subject: [PATCH 142/161] fix download + better curl flags --- scripts/build_windows.sh | 2 +- scripts/utils.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_windows.sh b/scripts/build_windows.sh index 1c5f19f0..4886b58d 100755 --- a/scripts/build_windows.sh +++ b/scripts/build_windows.sh @@ -85,7 +85,7 @@ echo "::endgroup::" echo "::group::zlib" cd ${WORKDIR} -download_verify_extract https://zlib.net/fossils/zlib-1.3.1.tar.gz +download_verify_extract zlib-1.3.1.tar.gz mkdir deps/zlib cd zlib-1.3.1 mkdir build diff --git a/scripts/utils.sh b/scripts/utils.sh index 98e2aa63..3f859600 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -12,7 +12,7 @@ function verify_checksum () { function download_and_verify () { file="$1" - curl -s -L -o $file https://github.com/bjia56/portable-python/releases/download/build-dependencies/$file + curl -s -S -f -L -o $file https://github.com/bjia56/portable-python/releases/download/build-dependencies/$file verify_checksum $file } From 15b8bbcc589c01f0dccef4df6f077ba8ca5bd454 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 14:36:15 -0500 Subject: [PATCH 143/161] unset compiler flags for cmake --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 894f9170..d9507d7a 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -412,7 +412,7 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -cmake \ +CFLAGS="" CPPFLAGS="" CXXFLAGS="" LDFLAGS="" PKG_CONFIG_PATH="" cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_C_STANDARD=99 \ From 6464dc850d8d6be28a87b621e3d788a636d853d5 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 14:57:39 -0500 Subject: [PATCH 144/161] Revert "unset compiler flags for cmake" This reverts commit 15b8bbcc589c01f0dccef4df6f077ba8ca5bd454. --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index d9507d7a..894f9170 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -412,7 +412,7 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -CFLAGS="" CPPFLAGS="" CXXFLAGS="" LDFLAGS="" PKG_CONFIG_PATH="" cmake \ +cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_C_STANDARD=99 \ From 4f4b0e9ba54e17434e73fa6a72c359ba6bc058f1 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 15:15:40 -0500 Subject: [PATCH 145/161] exclude system headers --- scripts/build_linux_zig.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 894f9170..f8ab605c 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -415,6 +415,7 @@ cd python-build cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ + -DCMAKE_IGNORE_PATH=/usr/include \ -DCMAKE_C_STANDARD=99 \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ -DCMAKE_BUILD_TYPE:STRING=Release \ From 79831391eb9121a5d35c319bbba2d85b59a7b07a Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 15:32:13 -0500 Subject: [PATCH 146/161] compile static tcl --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index f8ab605c..101ac59d 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -362,7 +362,7 @@ cd ${BUILDDIR} download_verify_extract tcl8.6.13-src.tar.gz cd tcl*/unix -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 33774a9c3990eee9f22943c9f121dae805f0b677 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 15:52:41 -0500 Subject: [PATCH 147/161] static tk --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 101ac59d..2d54de7b 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -375,7 +375,7 @@ cd ${BUILDDIR} download_verify_extract tk8.6.13-src.tar.gz cd tk*/unix -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 36b3c804808c487bbafd2433364c1dd9a5a2c861 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 16:11:14 -0500 Subject: [PATCH 148/161] static lib tweaks --- scripts/build_linux_zig.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 2d54de7b..7d1eb798 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -304,7 +304,7 @@ cd ${BUILDDIR} download_verify_extract fontconfig-2.15.0.tar.gz cd fontconfig* -./configure --host=${ARCH}-linux --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --enable-static --disable-shared --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} make -j4 make install @@ -452,7 +452,7 @@ cmake \ -DTCL_INCLUDE_PATH:FILEPATH=${DEPSDIR}/include/tcl.h \ -DTCL_LIBRARY:FILEPATH=${DEPSDIR}/lib/libtcl8.6.a \ -DX11_INCLUDE_DIR:PATH=${DEPSDIR}/include/X11 \ - -DX11_LIBRARIES="${DEPSDIR}/lib/libXau.a;${DEPSDIR}/lib/libXdmcp.a;${DEPSDIR}/lib/libX11.a;${DEPSDIR}/lib/libXext.a;${DEPSDIR}/lib/libICE.a;${DEPSDIR}/lib/libSM.a;${DEPSDIR}/lib/libXrender.a;${DEPSDIR}/lib/libXft.a;${DEPSDIR}/lib/libXss.a" \ + -DX11_LIBRARIES="${DEPSDIR}/lib/libXau.a;${DEPSDIR}/lib/libXdmcp.a;${DEPSDIR}/lib/libX11.a;${DEPSDIR}/lib/libXext.a;${DEPSDIR}/lib/libICE.a;${DEPSDIR}/lib/libSM.a;${DEPSDIR}/lib/libXrender.a;${DEPSDIR}/lib/libXft.a;${DEPSDIR}/lib/libXss.a;${DEPSDIR}/lib/libxcb.a" \ ../python-cmake-buildsystem make -j4 make install From 37731dc354ae17a1c979c2e68153957b688992bf Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 16:25:00 -0500 Subject: [PATCH 149/161] static libxml --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 7d1eb798..8ba86f33 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -231,7 +231,7 @@ cd ${BUILDDIR} download_verify_extract libxml2-2.12.4.tar.xz cd libxml2* -./configure --host=${ARCH}-linux --without-python --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --enable-static --disable-shared --without-python --prefix=${DEPSDIR} make -j4 make install From 65431f093eb764a257e24d7d5fcc6a7471c7d643 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 17:01:40 -0500 Subject: [PATCH 150/161] link static libxml --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 8ba86f33..ef9b026f 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -304,7 +304,7 @@ cd ${BUILDDIR} download_verify_extract fontconfig-2.15.0.tar.gz cd fontconfig* -./configure --host=${ARCH}-linux --enable-static --disable-shared --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} +LDFLAGS="${LDFLAGS} -l:libxml2.a" ./configure --host=${ARCH}-linux --enable-static --disable-shared --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} make -j4 make install From 8405637f4a551cdb687bf1deb0c8236d9b932f19 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 17:16:08 -0500 Subject: [PATCH 151/161] fontconfig without xml2 --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index ef9b026f..a32b3914 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -304,7 +304,7 @@ cd ${BUILDDIR} download_verify_extract fontconfig-2.15.0.tar.gz cd fontconfig* -LDFLAGS="${LDFLAGS} -l:libxml2.a" ./configure --host=${ARCH}-linux --enable-static --disable-shared --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --enable-static --disable-shared --disable-cache-build --prefix=${DEPSDIR} make -j4 make install From f7d6fc6aa5381912c87c1089793115101b177e7d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 18:20:32 -0500 Subject: [PATCH 152/161] link expat --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index a32b3914..620efdd6 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -362,7 +362,7 @@ cd ${BUILDDIR} download_verify_extract tcl8.6.13-src.tar.gz cd tcl*/unix -./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} +LDFLAGS="${LDFLAGS} -lexpat" ./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 52e82d0bef78ef9dfe6541f6c445885fa2323151 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 18:37:05 -0500 Subject: [PATCH 153/161] Link expat --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 620efdd6..c44e9130 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -375,7 +375,7 @@ cd ${BUILDDIR} download_verify_extract tk8.6.13-src.tar.gz cd tk*/unix -./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} +LDFLAGS="${LDFLAGS} -lexpat" ./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 24df6d2eb1876bc8da00afef9caacda83ac7320b Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 19:04:31 -0500 Subject: [PATCH 154/161] Link fontconfig, freetype --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index c44e9130..c7b4f261 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -412,7 +412,7 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -cmake \ +LDFLAGS="${LDFLAGS} -lfontconfig -lfreetype" cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_IGNORE_PATH=/usr/include \ From 3529debe032785f7e2b50c0f1c388ae38a1b24dc Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 19:33:54 -0500 Subject: [PATCH 155/161] Add expat --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index c7b4f261..32451e57 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -412,7 +412,7 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -LDFLAGS="${LDFLAGS} -lfontconfig -lfreetype" cmake \ +LDFLAGS="${LDFLAGS} -lexpat -lfontconfig -lfreetype" cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_IGNORE_PATH=/usr/include \ From c31572c2f52b1aa4a9358df1e93aa1cebc37532c Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 20:40:50 -0500 Subject: [PATCH 156/161] disable shared expat --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 32451e57..085607c3 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -118,7 +118,7 @@ cd ${BUILDDIR} download_verify_extract expat-2.5.0.tar.gz cd expat* -./configure --host=${ARCH}-linux --prefix=${DEPSDIR} +./configure --host=${ARCH}-linux --disable-shared --prefix=${DEPSDIR} make -j4 make install From e0ab404ace294ae40fd04f8a3dcac2cf9ae70665 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 21:00:09 -0500 Subject: [PATCH 157/161] Remove expat from python --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 085607c3..277951fb 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -412,7 +412,7 @@ mv *python-cmake-buildsystem* python-cmake-buildsystem mkdir python-build mkdir python-install cd python-build -LDFLAGS="${LDFLAGS} -lexpat -lfontconfig -lfreetype" cmake \ +LDFLAGS="${LDFLAGS} -lfontconfig -lfreetype" cmake \ -DCMAKE_SYSTEM_PROCESSOR=${ARCH} \ -DCMAKE_CROSSCOMPILING_EMULATOR=${WORKDIR}/scripts/qemu_${ARCH}_interpreter \ -DCMAKE_IGNORE_PATH=/usr/include \ From 837edd21a955fcc247d60053250d018b20c0323b Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 21:28:43 -0500 Subject: [PATCH 158/161] Revert "fontconfig without xml2" This reverts commit 8405637f4a551cdb687bf1deb0c8236d9b932f19. --- scripts/build_linux_zig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 277951fb..43cfa9da 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -304,7 +304,7 @@ cd ${BUILDDIR} download_verify_extract fontconfig-2.15.0.tar.gz cd fontconfig* -./configure --host=${ARCH}-linux --enable-static --disable-shared --disable-cache-build --prefix=${DEPSDIR} +LDFLAGS="${LDFLAGS} -l:libxml2.a" ./configure --host=${ARCH}-linux --enable-static --disable-shared --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} make -j4 make install From ca4d4af0114e55f9e26bc5a1de3521f2ea5561cb Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 21:29:58 -0500 Subject: [PATCH 159/161] switch to xml2 --- scripts/build_linux_zig.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 43cfa9da..260d927e 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -304,7 +304,7 @@ cd ${BUILDDIR} download_verify_extract fontconfig-2.15.0.tar.gz cd fontconfig* -LDFLAGS="${LDFLAGS} -l:libxml2.a" ./configure --host=${ARCH}-linux --enable-static --disable-shared --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} +LDFLAGS="${LDFLAGS} -lxml2" ./configure --host=${ARCH}-linux --enable-static --disable-shared --enable-libxml2 --disable-cache-build --prefix=${DEPSDIR} make -j4 make install @@ -362,7 +362,7 @@ cd ${BUILDDIR} download_verify_extract tcl8.6.13-src.tar.gz cd tcl*/unix -LDFLAGS="${LDFLAGS} -lexpat" ./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} +LDFLAGS="${LDFLAGS} -lxml2" ./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install @@ -375,7 +375,7 @@ cd ${BUILDDIR} download_verify_extract tk8.6.13-src.tar.gz cd tk*/unix -LDFLAGS="${LDFLAGS} -lexpat" ./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} +LDFLAGS="${LDFLAGS} -lxml2" ./configure --disable-shared --host=${ARCH}-linux --prefix=${DEPSDIR} make -j4 make install From 059f50b8b6e3e5096f4940d559cf775193f536b2 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 21:55:16 -0500 Subject: [PATCH 160/161] only archive build dir if requested --- .github/workflows/build_python.yml | 12 ++++++++++-- .github/workflows/build_python_linux.yml | 8 ++++++-- .github/workflows/build_python_on_branch.yml | 4 ++++ scripts/build_linux.sh | 6 ------ scripts/build_linux_zig.sh | 6 ------ scripts/build_macos.sh | 6 ------ scripts/build_windows.sh | 6 ------ scripts/utils.sh | 10 +++++++++- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index cd70e84f..7f4e7e04 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -9,6 +9,9 @@ on: run_tests: required: false type: boolean + debug: + required: false + type: boolean workflow_call: inputs: python_version: @@ -17,9 +20,13 @@ on: run_tests: required: false type: boolean + debug: + required: false + type: boolean env: RUN_TESTS: ${{ inputs.run_tests }} + DEBUG_CI: ${{ inputs.debug }} jobs: build_linux: @@ -33,6 +40,7 @@ jobs: python_version: ${{ inputs.python_version }} arch: ${{ matrix.arch }} run_tests: ${{ inputs.run_tests }} + debug: ${{ inputs.debug }} build_windows: name: Windows ${{ inputs.python_version }} x86_64 (build) @@ -69,7 +77,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 - if: ${{ always() }} + if: ${{ always() && inputs.debug }} with: name: build-python-windows-x86_64-${{ inputs.python_version }} path: ./*python*.tar.gz @@ -123,7 +131,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 - if: ${{ always() }} + if: ${{ always() && inputs.debug }} with: name: build-python-darwin-universal2-${{ inputs.python_version }} path: ./*python*.tar.gz diff --git a/.github/workflows/build_python_linux.yml b/.github/workflows/build_python_linux.yml index abdcc20d..8b69bd24 100644 --- a/.github/workflows/build_python_linux.yml +++ b/.github/workflows/build_python_linux.yml @@ -12,11 +12,15 @@ on: run_tests: required: false type: boolean + debug: + required: false + type: boolean env: image_map: '{"x86_64": "debian:buster", "aarch64": "debian:buster", "armv7l": "debian:buster"}' platform_map: '{"x86_64": "linux/amd64", "aarch64": "linux/arm64/v8", "armv7l": "linux/arm/v7"}' RUN_TESTS: ${{ inputs.run_tests }} + DEBUG_CI: ${{ inputs.debug }} jobs: build_linux_zig: @@ -49,7 +53,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 - if: ${{ always() }} + if: ${{ always() && inputs.debug }} with: name: build-python-linux-${{ inputs.arch }}-${{ inputs.python_version }} path: ./*python*.tar.gz @@ -127,7 +131,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 - if: ${{ always() }} + if: ${{ always() && inputs.debug }} with: name: build-python-linux-${{ inputs.arch }}-${{ inputs.python_version }} path: ./*python*.tar.gz diff --git a/.github/workflows/build_python_on_branch.yml b/.github/workflows/build_python_on_branch.yml index 7ea3ce13..baa396da 100644 --- a/.github/workflows/build_python_on_branch.yml +++ b/.github/workflows/build_python_on_branch.yml @@ -11,6 +11,9 @@ on: run_tests: required: false type: boolean + debug: + required: false + type: boolean jobs: build: @@ -23,3 +26,4 @@ jobs: with: python_version: ${{ matrix.python_version }} run_tests: ${{ inputs.run_tests || false }} + debug: ${{ inputs.debug || false }} diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh index 5a6fec13..25188975 100755 --- a/scripts/build_linux.sh +++ b/scripts/build_linux.sh @@ -84,12 +84,6 @@ if [[ "${ARCH}" == "armv7l" ]]; then export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib fi -if [[ "${RUN_TESTS}" == "true" ]]; then - INSTALL_TEST="ON" -else - INSTALL_TEST="OFF" -fi - #cmake --trace-expand \ cmake \ -DPYTHON_VERSION=${PYTHON_FULL_VER} \ diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 260d927e..9fccd147 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -399,12 +399,6 @@ echo "::endgroup::" echo "::group::Python" cd ${BUILDDIR} -if [[ "${RUN_TESTS}" == "true" ]]; then - INSTALL_TEST="ON" -else - INSTALL_TEST="OFF" -fi - wget --no-verbose -O python-cmake-buildsystem.tar.gz https://github.com/bjia56/python-cmake-buildsystem/tarball/portable-python tar -xf python-cmake-buildsystem.tar.gz rm *.tar.gz diff --git a/scripts/build_macos.sh b/scripts/build_macos.sh index 44042e8e..1c7586d3 100755 --- a/scripts/build_macos.sh +++ b/scripts/build_macos.sh @@ -164,12 +164,6 @@ echo "::endgroup::" echo "::group::Build" cd ${WORKDIR} -if [[ "${RUN_TESTS}" == "true" ]]; then - INSTALL_TEST="ON" -else - INSTALL_TEST="OFF" -fi - cd python-build cmake \ -G "Unix Makefiles" \ diff --git a/scripts/build_windows.sh b/scripts/build_windows.sh index 4886b58d..f844c020 100755 --- a/scripts/build_windows.sh +++ b/scripts/build_windows.sh @@ -123,12 +123,6 @@ echo "::endgroup::" echo "::group::Build" cd ${WORKDIR} -if [[ "${RUN_TESTS}" == "true" ]]; then - INSTALL_TEST="ON" -else - INSTALL_TEST="OFF" -fi - cd python-build cmake \ -G "Visual Studio 17 2022" -A x64 \ diff --git a/scripts/utils.sh b/scripts/utils.sh index 3f859600..38c83d67 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -31,4 +31,12 @@ WORKDIR=$(pwd) BUILDDIR=${WORKDIR}/build DEPSDIR=${WORKDIR}/deps -trap "cd ${BUILDDIR} && tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-${PLATFORM}-${ARCH}.tar.gz ." EXIT +if [[ "${RUN_TESTS}" == "true" ]]; then + INSTALL_TEST="ON" +else + INSTALL_TEST="OFF" +fi + +if [[ "${DEBUG_CI}" == "true" ]]; then + trap "cd ${BUILDDIR} && tar -czf ${WORKDIR}/build-python-${PYTHON_FULL_VER}-${PLATFORM}-${ARCH}.tar.gz ." EXIT +fi From c389f604cb23ab2121f9a8b4d4e72017ae2f529d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 30 Jan 2024 22:36:18 -0500 Subject: [PATCH 161/161] copy in tcl/tk --- scripts/build_linux_zig.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/build_linux_zig.sh b/scripts/build_linux_zig.sh index 9fccd147..ea5e6269 100755 --- a/scripts/build_linux_zig.sh +++ b/scripts/build_linux_zig.sh @@ -451,6 +451,10 @@ LDFLAGS="${LDFLAGS} -lfontconfig -lfreetype" cmake \ make -j4 make install +cd ${BUILDDIR} +cp -r ${DEPSDIR}/lib/tcl8.6 ./python-install/lib +cp -r ${DEPSDIR}/lib/tk8.6 ./python-install/lib + echo "::endgroup::" ############################################# # Check executable dependencies (pre-patch) #