From acc75e75f119483a56fb46e06951004e8cfa5594 Mon Sep 17 00:00:00 2001 From: Pedro Tammela Date: Thu, 12 Sep 2024 15:13:07 -0300 Subject: [PATCH 1/2] ci: update musl headers to Linux 6.6 Update the musl headers in CI to use alpine-linux instead of sabotage-linux. Alpine also uses musl but follows the linux stable releases, providing more up-to-date headers. Signed-off-by: Pedro Tammela --- .../aarch64-unknown-linux-musl/Dockerfile | 2 +- .../arm-unknown-linux-musleabihf/Dockerfile | 2 +- ci/docker/i686-unknown-linux-musl/Dockerfile | 4 +- ci/docker/s390x-unknown-linux-musl/Dockerfile | 3 +- .../x86_64-unknown-linux-musl/Dockerfile | 3 +- ci/install-musl.sh | 66 ++++++++++++++++--- 6 files changed, 66 insertions(+), 14 deletions(-) diff --git a/ci/docker/aarch64-unknown-linux-musl/Dockerfile b/ci/docker/aarch64-unknown-linux-musl/Dockerfile index 65932bd48cfac..053ed837b2e7c 100644 --- a/ci/docker/aarch64-unknown-linux-musl/Dockerfile +++ b/ci/docker/aarch64-unknown-linux-musl/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:24.10 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc make libc6-dev git curl ca-certificates \ - gcc-aarch64-linux-gnu qemu-user + gcc-aarch64-linux-gnu qemu-user xz-utils patch rsync COPY install-musl.sh / RUN /install-musl.sh aarch64 diff --git a/ci/docker/arm-unknown-linux-musleabihf/Dockerfile b/ci/docker/arm-unknown-linux-musleabihf/Dockerfile index 7ed23611c351e..c6bd116b6f1cb 100644 --- a/ci/docker/arm-unknown-linux-musleabihf/Dockerfile +++ b/ci/docker/arm-unknown-linux-musleabihf/Dockerfile @@ -5,7 +5,7 @@ RUN sed -i -E 's/(archive|security)\.ubuntu\.com/old-releases.ubuntu.com/g' \ /etc/apt/sources.list && \ apt-get update && apt-get install -y --no-install-recommends \ gcc make libc6-dev git curl ca-certificates \ - gcc-arm-linux-gnueabihf qemu-user + gcc-arm-linux-gnueabihf qemu-user xz-utils patch rsync COPY install-musl.sh / RUN /install-musl.sh arm diff --git a/ci/docker/i686-unknown-linux-musl/Dockerfile b/ci/docker/i686-unknown-linux-musl/Dockerfile index ea5e2e963910b..287f325e9151f 100644 --- a/ci/docker/i686-unknown-linux-musl/Dockerfile +++ b/ci/docker/i686-unknown-linux-musl/Dockerfile @@ -1,12 +1,12 @@ FROM ubuntu:23.10 - # FIXME(time): we are using an EOL release because 24.04 changes to 64-bit time RUN sed -i -E 's/(archive|security)\.ubuntu\.com/old-releases.ubuntu.com/g' \ /etc/apt/sources.list && \ dpkg --add-architecture i386 && \ apt-get update && apt-get install -y --no-install-recommends \ - gcc-multilib make libc6-dev git curl ca-certificates libc6-i386 + gcc-multilib make libc6-dev git curl ca-certificates libc6-i386 \ + xz-utils patch rsync COPY install-musl.sh / RUN /install-musl.sh i686 diff --git a/ci/docker/s390x-unknown-linux-musl/Dockerfile b/ci/docker/s390x-unknown-linux-musl/Dockerfile index 4e202d1905902..2d4ea759c5fbf 100644 --- a/ci/docker/s390x-unknown-linux-musl/Dockerfile +++ b/ci/docker/s390x-unknown-linux-musl/Dockerfile @@ -4,7 +4,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl ca-certificates \ gcc \ gcc-s390x-linux-gnu \ - qemu-user + qemu-user \ + xz-utils patch rsync COPY install-musl.sh / RUN /install-musl.sh s390x diff --git a/ci/docker/x86_64-unknown-linux-musl/Dockerfile b/ci/docker/x86_64-unknown-linux-musl/Dockerfile index d03df5b4f54ce..5c1b4b177880c 100644 --- a/ci/docker/x86_64-unknown-linux-musl/Dockerfile +++ b/ci/docker/x86_64-unknown-linux-musl/Dockerfile @@ -2,7 +2,8 @@ FROM ubuntu:24.10 RUN apt-get update RUN apt-get install -y --no-install-recommends \ - gcc make libc6-dev git curl ca-certificates + gcc make libc6-dev git curl ca-certificates \ + xz-utils patch rsync COPY install-musl.sh / RUN /install-musl.sh x86_64 diff --git a/ci/install-musl.sh b/ci/install-musl.sh index 5f8c681fa6678..1cf1ec6500cde 100755 --- a/ci/install-musl.sh +++ b/ci/install-musl.sh @@ -64,13 +64,63 @@ esac cd .. rm -rf "$musl" -# Download, configure, build, and install musl-sanitized kernel headers: -kernel_header_ver="4.19.88" -curl --retry 5 -L \ - "https://github.com/sabotage-linux/kernel-headers/archive/v${kernel_header_ver}.tar.gz" | - tar xzf - +# Download, configure, build, and install musl-sanitized kernel headers. + +# Alpine follows stable kernel releases, 3.20 uses Linux 6.6 headers. +alpine_version=3.20 +alpine_git=https://gitlab.alpinelinux.org/alpine/aports + +# This routine piggybacks on: https://git.alpinelinux.org/aports/tree/main/linux-headers?h=3.20-stable +git clone -n --depth=1 --filter=tree:0 -b "${alpine_version}-stable" "$alpine_git" ( - cd "kernel-headers-${kernel_header_ver}" - make ARCH="${kernel_arch}" prefix="/musl-${musl_arch}" install -j4 + cd aports + git sparse-checkout set --no-cone main/linux-headers + git checkout + + cd main/linux-headers + cp APKBUILD APKBUILD.vars + cat <<- EOF >> APKBUILD.vars + echo "\$source" > alpine-source + echo "\$_kernver" > alpine-kernver + echo "\$pkgver" > alpine-pkgver + echo "\$sha512sums" > alpine-sha512sums +EOF + + # Retrieve all the variables + sh APKBUILD.vars + + cat APKBUILD.vars + + kernel_version=$(tr -d "[:space:]" < alpine-kernver) + pkg_version=$(tr -d "[:space:]" < alpine-pkgver) + + urls=$(grep -o 'https.*' alpine-source) + kernel="" + patch="" + for url in $urls; do + base=$(basename "$url") + curl --retry 5 -L "$url" > "$base" + case $base in + linux-*) kernel=$base;; + patch-*) patch=$base;; + esac + # Check if file is known + grep -o "$base" alpine-sha512sums + done + + # Double check checksums + sha512sum -c alpine-sha512sums + + # Extract, apply patches, compile and install headers + tar -xf "$kernel" + cd "linux-$kernel_version" + if [ "$pkg_version" != "$kernel_version" ]; then + unxz -c < "../$patch" | patch -p1 + fi + for p in ../*.patch; do + patch -p1 < "$p" + done + make headers_install ARCH="${kernel_arch}" INSTALL_HDR_PATH="/musl-${musl_arch}" ) -rm -rf kernel-headers-${kernel_header_ver} + +rm -rf aports From 55f945132b70208bdeb1fea2e552cb987929fe5b Mon Sep 17 00:00:00 2001 From: Pedro Tammela Date: Fri, 13 Sep 2024 12:01:21 -0300 Subject: [PATCH 2/2] ci: clean up musl exceptions Now that we have Linux 6.6 we can clean up some of the test exceptions. Not all of them can be removed, so the comments are updated if needed. Signed-off-by: Pedro Tammela --- libc-test/build.rs | 164 ++++----------------------------------------- 1 file changed, 12 insertions(+), 152 deletions(-) diff --git a/libc-test/build.rs b/libc-test/build.rs index b352add16b327..5aa5636b953ee 100644 --- a/libc-test/build.rs +++ b/libc-test/build.rs @@ -3505,8 +3505,7 @@ fn test_linux(target: &str) { [gnu]: "linux/aio_abi.h", "linux/can.h", "linux/can/raw.h", - // FIXME: requires kernel headers >= 5.4.1. - [!musl]: "linux/can/j1939.h", + "linux/can/j1939.h", "linux/dccp.h", "linux/errqueue.h", "linux/falloc.h", @@ -3531,8 +3530,7 @@ fn test_linux(target: &str) { "linux/mempolicy.h", "linux/mman.h", "linux/module.h", - // FIXME: requires kernel headers >= 5.1. - [!musl]: "linux/mount.h", + "linux/mount.h", "linux/net_tstamp.h", "linux/netfilter/nfnetlink.h", "linux/netfilter/nfnetlink_log.h", @@ -3544,11 +3542,10 @@ fn test_linux(target: &str) { "linux/netfilter_ipv6.h", "linux/netfilter_ipv6/ip6_tables.h", "linux/netlink.h", - // FIXME: requires Linux >= 5.6: - [!musl]: "linux/openat2.h", + "linux/openat2.h", // FIXME: some items require Linux >= 5.6: "linux/ptp_clock.h", - [!musl]: "linux/ptrace.h", + "linux/ptrace.h", "linux/quota.h", "linux/random.h", "linux/reboot.h", @@ -3566,7 +3563,7 @@ fn test_linux(target: &str) { "sys/fanotify.h", // is not present on uclibc [!uclibc]: "sys/auxv.h", - [gnu]: "linux/close_range.h", + [gnu || musl]: "linux/close_range.h", } // note: aio.h must be included before sys/mount.h @@ -3657,11 +3654,6 @@ fn test_linux(target: &str) { // specific type. "Ioctl" => true, - // FIXME: requires >= 5.4.1 kernel headers - "pgn_t" if musl => true, - "priority_t" if musl => true, - "name_t" if musl => true, - // FIXME: "'__uint128' undeclared" in C "__uint128" => true, @@ -3680,22 +3672,6 @@ fn test_linux(target: &str) { if ty.starts_with("__c_anonymous_") { return true; } - // FIXME: musl CI has old headers - if musl && ty.starts_with("uinput_") { - return true; - } - if musl && ty == "seccomp_notif" { - return true; - } - if musl && ty == "seccomp_notif_addfd" { - return true; - } - if musl && ty == "seccomp_notif_resp" { - return true; - } - if musl && ty == "seccomp_notif_sizes" { - return true; - } // FIXME: CI has old headers if ty == "ptp_sys_offset_extended" { @@ -3779,12 +3755,6 @@ fn test_linux(target: &str) { // Might differ between kernel versions "open_how" => true, - // FIXME: requires >= 5.4.1 kernel headers - "j1939_filter" if musl => true, - - // FIXME: requires >= 5.4 kernel headers - "sockaddr_can" if musl => true, - "sctp_initmsg" | "sctp_sndrcvinfo" | "sctp_sndinfo" | "sctp_rcvinfo" | "sctp_nxtinfo" | "sctp_prinfo" | "sctp_authinfo" => true, @@ -3949,7 +3919,7 @@ fn test_linux(target: &str) { return true; } // FIXME: Requires >= 6.3 kernel headers - if name == "MFD_NOEXEC_SEAL" || name == "MFD_EXEC" { + if loongarch64 && (name == "MFD_NOEXEC_SEAL" || name == "MFD_EXEC") { return true; } } @@ -4036,7 +4006,7 @@ fn test_linux(target: &str) { "SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV" if sparc64 => true, // FIXME: Not currently available in headers on ARM and musl. - "NETLINK_GET_STRICT_CHK" if arm || musl => true, + "NETLINK_GET_STRICT_CHK" if arm => true, // kernel constants not available in uclibc 1.0.34 | "EXTPROC" @@ -4105,62 +4075,14 @@ fn test_linux(target: &str) { | "MINSIGSTKSZ" if gnu => true, - // FIXME: Linux >= 5.10: - // https://github.com/torvalds/linux/commit/d25e2e9388eda61b6e298585024ee3355f50c493 - "NF_INET_INGRESS" if musl => true, - // FIXME: Linux >= 5.16: // https://github.com/torvalds/linux/commit/42df6e1d221dddc0f2acf2be37e68d553ad65f96 - "NF_NETDEV_EGRESS" if musl || sparc64 => true, + "NF_NETDEV_EGRESS" if sparc64 => true, // value changed - "NF_NETDEV_NUMHOOKS" if musl || sparc64 => true, - - // FIXME: requires Linux >= 5.6: - | "RESOLVE_BENEATH" - | "RESOLVE_CACHED" - | "RESOLVE_IN_ROOT" - | "RESOLVE_NO_MAGICLINKS" - | "RESOLVE_NO_SYMLINKS" - | "RESOLVE_NO_XDEV" if musl => true, - - // FIXME: requires Linux >= 5.4: - | "CAN_J1939" - | "CAN_NPROTO" if musl => true, - - // FIXME: requires Linux >= 5.6 - "GRND_INSECURE" if musl => true, - - // FIXME: requires Linux >= 5.7: - "MREMAP_DONTUNMAP" if musl => true, + "NF_NETDEV_NUMHOOKS" if sparc64 => true, // FIXME: requires Linux >= v5.8 - "IF_LINK_MODE_TESTING" if musl || sparc64 => true, - - // FIXME: Requires more recent kernel headers (5.9 / 5.11): - | "CLOSE_RANGE_UNSHARE" - | "CLOSE_RANGE_CLOEXEC" if musl => true, - - // FIXME: requires Linux >= 5.12: - "MPOL_F_NUMA_BALANCING" if musl => true, - - // FIXME: Requires more recent kernel headers - | "NFNL_SUBSYS_COUNT" // bumped in v5.14 - | "NFNL_SUBSYS_HOOK" // v5.14+ - | "NFULA_VLAN" // v5.4+ - | "NFULA_L2HDR" // v5.4+ - | "NFULA_VLAN_PROTO" // v5.4+ - | "NFULA_VLAN_TCI" // v5.4+ - | "NFULA_VLAN_UNSPEC" // v5.4+ - | "RTNLGRP_NEXTHOP" // linux v5.3+ - | "RTNLGRP_BRVLAN" // linux v5.6+ - if musl => true, - - | "MADV_COLD" - | "MADV_PAGEOUT" - | "MADV_POPULATE_READ" - | "MADV_POPULATE_WRITE" - if musl => true, - "CLONE_CLEAR_SIGHAND" | "CLONE_INTO_CGROUP" => true, + "IF_LINK_MODE_TESTING" if sparc64 => true, // FIXME: Requires >= 6.3 kernel headers "MFD_EXEC" | "MFD_NOEXEC_SEAL" if sparc64 => true, @@ -4182,9 +4104,6 @@ fn test_linux(target: &str) { => true, "SCTP_FUTURE_ASSOC" | "SCTP_CURRENT_ASSOC" | "SCTP_ALL_ASSOC" | "SCTP_PEER_ADDR_THLDS_V2" => true, // linux 5.5+ - // FIXME: Requires more recent kernel headers - "HWTSTAMP_TX_ONESTEP_P2P" if musl => true, // linux v5.6+ - // kernel 6.5 minimum "MOVE_MOUNT_BENEATH" => true, // FIXME: Requires linux 6.1 @@ -4206,10 +4125,8 @@ fn test_linux(target: &str) { | "FAN_INFO" // linux v5.16+ => true, - // FIXME: Requires linux 5.15+ - "FAN_REPORT_PIDFD" if musl => true, - - // FIXME: Requires linux 5.9+ + // musl doesn't use in + "FAN_REPORT_PIDFD" | "FAN_REPORT_DIR_FID" | "FAN_REPORT_NAME" | "FAN_REPORT_DFID_NAME" @@ -4223,55 +4140,6 @@ fn test_linux(target: &str) { // FIXME: Requires linux 6.5 "NFT_MSG_MAX" => true, - // FIXME: Requires >= 5.1 kernel headers. - // Everything that uses install-musl.sh has 4.19 kernel headers. - "TLS_1_3_VERSION" - | "TLS_1_3_VERSION_MAJOR" - | "TLS_1_3_VERSION_MINOR" - | "TLS_CIPHER_AES_GCM_256" - | "TLS_CIPHER_AES_GCM_256_IV_SIZE" - | "TLS_CIPHER_AES_GCM_256_KEY_SIZE" - | "TLS_CIPHER_AES_GCM_256_SALT_SIZE" - | "TLS_CIPHER_AES_GCM_256_TAG_SIZE" - | "TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE" - if (aarch64 || arm || i686 || s390x || x86_64) && musl => - { - true - } - - // FIXME: Requires >= 5.11 kernel headers. - // Everything that uses install-musl.sh has 4.19 kernel headers. - "TLS_CIPHER_CHACHA20_POLY1305" - | "TLS_CIPHER_CHACHA20_POLY1305_IV_SIZE" - | "TLS_CIPHER_CHACHA20_POLY1305_KEY_SIZE" - | "TLS_CIPHER_CHACHA20_POLY1305_SALT_SIZE" - | "TLS_CIPHER_CHACHA20_POLY1305_TAG_SIZE" - | "TLS_CIPHER_CHACHA20_POLY1305_REC_SEQ_SIZE" - if (aarch64 || arm || i686 || s390x || x86_64) && musl => - { - true - } - - // FIXME: Requires >= 5.3 kernel headers. - // Everything that uses install-musl.sh has 4.19 kernel headers. - "XDP_OPTIONS_ZEROCOPY" | "XDP_OPTIONS" - if musl => - { - true - } - - // FIXME: Requires >= 5.4 kernel headers. - // Everything that uses install-musl.sh has 4.19 kernel headers. - "XSK_UNALIGNED_BUF_OFFSET_SHIFT" - | "XSK_UNALIGNED_BUF_ADDR_MASK" - | "XDP_UMEM_UNALIGNED_CHUNK_FLAG" - | "XDP_RING_NEED_WAKEUP" - | "XDP_USE_NEED_WAKEUP" - if musl => - { - true - } - // FIXME: Requires >= 6.6 kernel headers. "XDP_USE_SG" | "XDP_PKT_CONTD" @@ -4330,14 +4198,6 @@ fn test_linux(target: &str) { | "PF_MCE_EARLY" | "PF_MEMALLOC_PIN" => true, - "SCHED_FLAG_KEEP_POLICY" - | "SCHED_FLAG_KEEP_PARAMS" - | "SCHED_FLAG_UTIL_CLAMP_MIN" - | "SCHED_FLAG_UTIL_CLAMP_MAX" - | "SCHED_FLAG_KEEP_ALL" - | "SCHED_FLAG_UTIL_CLAMP" - | "SCHED_FLAG_ALL" if musl => true, // Needs more recent linux headers. - // FIXME: Requires >= 6.9 kernel headers. "EPIOCSPARAMS" | "EPIOCGPARAMS" => true,