Make libpq v16 default and updates

- Updated libpq to v15.8 and added v16.4
BlackDex · Aug 8, 2024 · bde4610 · bde4610
1 parent 3a51047
commit bde4610
Show file tree

Hide file tree

Showing 4 changed files with 107 additions and 7 deletions.
diff --git a/.github/workflows/rust-musl.yml b/.github/workflows/rust-musl.yml
@@ -410,6 +410,21 @@ jobs:
             -e RUSTFLAGS="${{ matrix.env.XTRA_RUSTFLAGS }}-Clink-arg=-s" \
             blackdex/rust-musl:${{ matrix.env.IMAGE_TAG }}-test bash -c 'rm -vf Cargo.lock ; cargo -Vv ; rustc -Vv ; cargo update ; cargo build --release'
 
+      - name: Test Docker Image (PQ11) - Rust Current Stable
+        # Skip during nightly builds
+        if: ${{ needs.build_vars.outputs.stable_trigger || !needs.build_vars.outputs.nightly_trigger }}
+        # continue-on-error: true
+        run: |
+          # Run the test
+          docker run --rm \
+            -v cargo-cache-${{ matrix.env.IMAGE_TAG }}:/root/.cargo/registry \
+            -v "$(pwd)/test/multicrate":/home/rust/src \
+            --tmpfs /home/rust/src/target:rw,exec,mode=1777 \
+            -e RUST_BACKTRACE=1 \
+            -e PQ_LIB_DIR="/usr/local/musl/pq11/lib" \
+            -e RUSTFLAGS="${{ matrix.env.XTRA_RUSTFLAGS }}-Clink-arg=-s" \
+            blackdex/rust-musl:${{ matrix.env.IMAGE_TAG }}-test bash -c 'rm -vf Cargo.lock ; cargo -Vv ; rustc -Vv ; cargo update ; cargo build --release'
+
       - name: Docker Push - Rust Current Stable
         # Skip during nightly builds
         if: ${{ needs.build_vars.outputs.stable_trigger || !needs.build_vars.outputs.nightly_trigger }}
@@ -482,6 +497,21 @@ jobs:
             -e RUSTFLAGS="${{ matrix.env.XTRA_RUSTFLAGS }}-Clink-arg=-s" \
             blackdex/rust-musl:${{ matrix.env.IMAGE_TAG }}-vw-test bash -c 'rm -vf Cargo.lock ; cargo -Vv ; rustc -Vv ; cargo update ; cargo build --release'
 
+      - name: Test Docker Image (PQ11) - Rust Vaultwarden Stable
+        # Skip during nightly builds
+        if: ${{ needs.build_vars.outputs.stable_version != needs.build_vars.outputs.vaultwarden_version && (needs.build_vars.outputs.stable_trigger || !needs.build_vars.outputs.nightly_trigger) }}
+        # continue-on-error: true
+        run: |
+          # Run the test
+          docker run --rm \
+            -v cargo-cache:/root/.cargo/registry \
+            -v "$(pwd)/test/multicrate":/home/rust/src \
+            --tmpfs /home/rust/src/target:rw,exec,mode=1777 \
+            -e RUST_BACKTRACE=1 \
+            -e PQ_LIB_DIR="/usr/local/musl/pq11/lib" \
+            -e RUSTFLAGS="${{ matrix.env.XTRA_RUSTFLAGS }}-Clink-arg=-s" \
+            blackdex/rust-musl:${{ matrix.env.IMAGE_TAG }}-vw-test bash -c 'rm -vf Cargo.lock ; cargo -Vv ; rustc -Vv ; cargo update ; cargo build --release'
+
       - name: Docker Push - Rust Vaultwarden Stable
         # Skip during nightly builds
         if: ${{ needs.build_vars.outputs.stable_version != needs.build_vars.outputs.vaultwarden_version && (needs.build_vars.outputs.stable_trigger || !needs.build_vars.outputs.nightly_trigger) }}
@@ -547,6 +577,19 @@ jobs:
             -e RUSTFLAGS="${{ matrix.env.XTRA_RUSTFLAGS }}-Clink-arg=-s" \
             blackdex/rust-musl:${{ matrix.env.IMAGE_TAG }}-nightly-test bash -c 'rm -vf Cargo.lock ; cargo -Vv ; rustc -Vv ; cargo update ; cargo build --release'
 
+      - name: Test Docker Image (PQ11) - Rust Nightly
+        # continue-on-error: true
+        run: |
+          # Run the test
+          docker run --rm \
+            -v cargo-cache:/root/.cargo/registry \
+            -v "$(pwd)/test/multicrate":/home/rust/src \
+            --tmpfs /home/rust/src/target:rw,exec,mode=1777 \
+            -e RUST_BACKTRACE=1 \
+            -e PQ_LIB_DIR="/usr/local/musl/pq11/lib" \
+            -e RUSTFLAGS="${{ matrix.env.XTRA_RUSTFLAGS }}-Clink-arg=-s" \
+            blackdex/rust-musl:${{ matrix.env.IMAGE_TAG }}-nightly-test bash -c 'rm -vf Cargo.lock ; cargo -Vv ; rustc -Vv ; cargo update ; cargo build --release'
+
       - name: Docker Push - Rust Nightly
         uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v6.5.0
         with:

diff --git a/Dockerfile.musl-base b/Dockerfile.musl-base
@@ -73,7 +73,8 @@ ENV TARGET_PKG_CONFIG_PATH="${TARGET_PREFIX}/lib/pkgconfig" \
     CURL_VER="8.9.1" \
     ZLIB_VER="1.3.1" \
     PQ_11_VER="11.22" \
-    PQ_15_VER="15.7" \
+    PQ_15_VER="15.8" \
+    PQ_16_VER="16.4" \
     SQLITE_VER="3460000" \
     MARIADB_VER="3.3.10" \
     LIBXML2_VER="2.13.3"
@@ -294,6 +295,58 @@ RUN echo "PostgreSQL v15" && \
     # Default cleanups
     find /var/log -type f -delete && rm -rf "${TARGET_PREFIX}/pq15/share/man"
 
+# Build libpq v16
+# hadolint ignore=DL3003
+RUN echo "PostgreSQL v16" && \
+    curl -sSL "https://ftp.postgresql.org/pub/source/v${PQ_16_VER}/postgresql-${PQ_16_VER}.tar.gz" | tar xz && \
+    cd "postgresql-${PQ_16_VER}" && \
+    if [[ "${TARGET}" == "arm-unknown-linux-musleabi" ]] ; then export LIBS="-latomic" ; fi && \
+    AR="${TARGET_AR}" \
+    LD="${TARGET_LD}" \
+    RANLIB="${TARGET_RANLIB}" \
+    LDFLAGS="${TARGET_LDFLAGS}" \
+    CPPFLAGS="${SHARED_CPPFLAGS}" \
+    CFLAGS="${SHARED_CFLAGS} -flto=auto -ffat-lto-objects" \
+    CC="${TARGET_CC}" \
+   ./configure \
+      --host="${TARGET}" \
+      --target="${TARGET}" \
+      --without-readline \
+      --with-ssl=openssl \
+      --disable-rpath \
+      --without-icu \
+      --with-system-tzdata=/usr/share/zoneinfo \
+      --prefix="${TARGET_PREFIX}/pq16" && \
+    # build common, port and libpq only
+    cd src/common && \
+      make -j"$(nproc)" all && \
+      make -j"$(nproc)" install && \
+    cd ../../src/port && \
+      make -j"$(nproc)" all && \
+      make -j"$(nproc)" install && \
+    cd ../../src/interfaces/libpq && \
+      make -j"$(nproc)" all-static-lib && \
+      make -j"$(nproc)" install-lib-static && \
+    # Merge these libraries so pq-sys will work without using pkg-config
+    cd "${TARGET_PREFIX}/pq16/lib" && \
+      mkdir -v merge && cd merge && \
+      # Extract the generated .a files
+      "${TARGET_AR}" x ../libpq.a && \
+      "${TARGET_AR}" x ../libpgcommon.a && \
+      "${TARGET_AR}" x ../libpgport.a && \
+      # Merge all these files again into one libpq.a file
+      # This is needed because the pq-sys crate only checks this file: https://github.com/sgrif/pq-sys/issues/27
+      "${TARGET_AR}" cvq libpq.a ./*.o && \
+      # Move and cleanup
+      mv -vf libpq.a ../libpq.a && \
+      cd ../ && \
+      "${TARGET_AR}" sv libpq.a && \
+      rm -rf merge libpgcommon* libpgport* && \
+    # Remove the source
+    cd /tmp && rm -rf "postgresql-${PQ_16_VER}" && \
+    # Default cleanups
+    find /var/log -type f -delete && rm -rf "${TARGET_PREFIX}/pq16/share/man"
+
 # Build libsqlite3
 # hadolint ignore=DL3003
 RUN echo "SQLite3" && \
@@ -495,7 +548,7 @@ ENV HOST="x86_64-unknown-linux-gnu" \
     TARGET_PKG_CONFIG_LIBDIR="${PKG_CONFIG_LIBDIR}" \
     # pq-sys (PosgreSQL) support
     PQ_LIB_STATIC=1 \
-    PQ_LIB_DIR="${TARGET_PREFIX}/pq11/lib" \
+    PQ_LIB_DIR="${TARGET_PREFIX}/pq15/lib" \
     # openssl-sys support
     OPENSSL_STATIC=1 \
     OPENSSL_DIR="${TARGET_PREFIX}" \

diff --git a/README.md b/README.md
@@ -17,7 +17,7 @@ The following libraries are pre-build and marked as `STATIC` already via `ENV` v
 * OpenSSL (`v3.0.14`)
 * cURL (`v8.9.1`)
 * ZLib (`v1.3.1`)
-* PostgreSQL lib (`v11.22`) and (`v15.7`)
+* PostgreSQL lib (`v16.4`) and (`v15.8`) and legacy (`v11.22`)
 * SQLite (`v3.46.0`)
 * MariaDB Connector/C (`v3.3.10`) (MySQL Compatible)
 * libxml2 (`v2.13.3`)
@@ -40,16 +40,19 @@ Now only OpenSSL v3.0 is being build.
 Since 2024-03-15 I stopped adding the `-openssl3` postfix to the tags.
 
 
-### PostgreSQL v15
-The default PostgreSQL lib used is v11, this might change in the future.<br>
-If you want to use v15 you need to overwrite an environment variable so that the postgresql crate will look at the right directory.<br>
+### PostgreSQL v16 & v15 (and legacy v11)
+The default PostgreSQL lib used is v16.<br>
+If you want to use v16 or legacy v11 you need to overwrite an environment variable so that the postgresql crate will look at the right directory.<br>
 <br>
-Adding `-e PQ_LIB_DIR="/usr/local/musl/pq15/lib"` at the cli or `ENV PQ_LIB_DIR="/usr/local/musl/pq15/lib"` to your custom build image will trigger the v15 version to be used during the build.
+Adding `-e PQ_LIB_DIR="/usr/local/musl/pq15/lib"` at the cli or `ENV PQ_LIB_DIR="/usr/local/musl/pq15/lib"` to your custom build image will trigger the v15 version to be used during the build. The same goes for using the legacy `v11`, just use `ENV PQ_LIB_DIR="/usr/local/musl/pq11/lib"`.
 
 **NOTE** 2024-08-02:<br>
 In some situations it could be that the libpq v11 was still used. Depending if during the compilation of the code other crates added the main library path as a search path after `pq-sys` did, which caused rustc to use a different libpq.a.<br>
 This has been solved now by moving the library file for v11 to a separate directory also. The default directory is changed and should not cause any issues unless you set the `PQ_LIB_DIR` variable your self to anything else then the v15 directory.
 
+**NOTE** 2024-08-08:<br>
+libpq v16 is now the default version. v15 and v11 are still build and available.
+
 <br>
 
 ## Usage

diff --git a/update_libs.py b/update_libs.py
@@ -157,6 +157,7 @@ def libxml2ver(site):
         'ZLIB': mirrorver('https://zlib.net/', r'zlib-\d\.\d+', r'zlib-', r'\.tar\.gz'),
         'PQ_11': mirrorver('https://ftp.postgresql.org/pub/source/', r'v11\.', 'v'),
         'PQ_15': mirrorver('https://ftp.postgresql.org/pub/source/', r'v15\.', 'v'),
+        'PQ_16': mirrorver('https://ftp.postgresql.org/pub/source/', r'v16\.', 'v'),
         'SQLITE': convert_sqlite_version(mirrorver('https://www.sqlite.org/chronology.html', r'releaselog\/\d_\d+\_\d+', r'releaselog/', r'\.html')),
         'MARIADB': mirrorver('https://archive.mariadb.org/?C=M&O=D', r'connector-c-3\.\d+\.', 'connector-c-', r'\/'),
         'LIBXML2': libxml2ver('https://download.gnome.org/sources/libxml2/cache.json'),