bazel: add a sysroot for a more deterministic build

Provide a sysroot from an ubuntu jammy docker image to provide a consistent set of headers, and system libraries for us to use to build Redpanda. This way there should be less differences between machines and we can remove a couple of required packages for development.
redpanda-data · Jan 3, 2025 · 525540b · 525540b
1 parent 2d07fa8
commit 525540b
Show file tree

Hide file tree

Showing 9 changed files with 134 additions and 12 deletions.
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -99,6 +99,8 @@ use_repo(non_module_dependencies, "seastar")
 use_repo(non_module_dependencies, "unordered_dense")
 use_repo(non_module_dependencies, "wasmtime")
 use_repo(non_module_dependencies, "xxhash")
+use_repo(non_module_dependencies, "x86_64_sysroot")
+use_repo(non_module_dependencies, "aarch64_sysroot")
 
 bazel_dep(name = "toolchains_llvm", version = "1.1.2")
 single_version_override(
@@ -131,6 +133,16 @@ llvm.toolchain(
         "linux-x86_64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/llvm-19.1.1-ubuntu-22.04-x86_64.tar.gz"],
     },
 )
+llvm.sysroot(
+    name = "llvm_19_toolchain",
+    label = "@x86_64_sysroot//:sysroot",
+    targets = ["linux-x86_64"],
+)
+llvm.sysroot(
+    name = "llvm_19_toolchain",
+    label = "@aarch64_sysroot//:sysroot",
+    targets = ["linux-aarch64"],
+)
 use_repo(llvm, "llvm_19_toolchain", "llvm_19_toolchain_llvm")
 
 register_toolchains("@llvm_19_toolchain//:all")
@@ -148,6 +160,17 @@ llvm.toolchain(
         "linux-x86_64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/pgo/llvm-18.1.8-ubuntu-22.04-x86_64.tar.gz"],
     },
 )
+
+llvm.sysroot(
+    name = "llvm_18_toolchain",
+    label = "@x86_64_sysroot//:sysroot",
+    targets = ["linux-x86_64"],
+)
+llvm.sysroot(
+    name = "llvm_18_toolchain",
+    label = "@aarch64_sysroot//:sysroot",
+    targets = ["linux-aarch64"],
+)
 use_repo(llvm, "llvm_18_toolchain", "llvm_18_toolchain_llvm")
 
 register_toolchains("@llvm_18_toolchain//:all")

diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
diff --git a/bazel/install-deps.sh b/bazel/install-deps.sh
@@ -34,8 +34,6 @@ deb_deps=(
   make
   pkgconf
   ragel
-  valgrind
-  xfslibs-dev
   sudo
 )
 
@@ -48,8 +46,6 @@ fedora_deps=(
   libtool
   perl
   ragel
-  valgrind-devel
-  xfsprogs-devel
   xorg-x11-util-macros
   sudo
 )

diff --git a/bazel/packaging/BUILD b/bazel/packaging/BUILD
@@ -102,7 +102,7 @@ pkg_tar(
     name = "redpanda_image_writable_dirs",
     empty_dirs = [
         "/etc/redpanda",
-        "/var/lib/redpanda",
+        "/var/lib/redpanda/data",
     ],
     mode = "0o755",
     owner = NONROOT_USER + "." + NONROOT_USER,

diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -197,3 +197,23 @@ def data_dependency():
         strip_prefix = "xxHash-bbb27a5efb85b92a0486cf361a8635715a53f6ba",
         url = "https://github.com/Cyan4973/xxHash/archive/bbb27a5efb85b92a0486cf361a8635715a53f6ba.tar.gz",
     )
+
+    sysroot_build_file = """
+filegroup(
+  name = "sysroot",
+  srcs = glob(["*/**"]),
+  visibility = ["//visibility:public"],
+)"""
+    http_archive(
+        name = "x86_64_sysroot",
+        build_file_content = sysroot_build_file,
+        sha256 = "b7544f6931531eababde47d8723362bc9b3c9de8ef7d9a5d1d62d2e614904d5b",
+        urls = ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/sysroot-ubuntu-22.04-x86_64-2025-01-03.tar.gz"],
+    )
+
+    http_archive(
+        name = "aarch64_sysroot",
+        build_file_content = sysroot_build_file,
+        sha256 = "a2d8c3ce82c70f884281db846e756f5877f2c77964cbfb1ba6b954e1f759e2ed",
+        urls = ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/sysroot-ubuntu-22.04-aarch64-2025-01-03.tar.gz"],
+    )
diff --git a/bazel/toolchain/.gitignore b/bazel/toolchain/.gitignore
@@ -1 +1 @@
-llvm-*.tar.gz
+*.tar.gz
diff --git a/bazel/toolchain/Dockerfile.llvm b/bazel/toolchain/Dockerfile.llvm
@@ -14,7 +14,7 @@ ADD --chmod=755 https://github.com/kadwanev/retry/raw/20997c7712a4/retry /usr/bi
 
 WORKDIR /opt
 
-ARG LLVM_VERSION=17.0.6
+ARG LLVM_VERSION=19.1.6
 
 RUN git clone --branch llvmorg-${LLVM_VERSION} --single-branch https://github.com/llvm/llvm-project.git
 
@@ -43,7 +43,7 @@ cmake -G Ninja -DLLVM_TARGETS_TO_BUILD='AArch64;X86' \\
   -DLLVM_INCLUDE_EXAMPLES=OFF \\
   -DLLVM_INCLUDE_UTILS=OFF \\
   -DLLVM_INCLUDE_DOCS=OFF \\
-  -DLLVM_ENABLE_ZLIB=OFF \\
+  -DLLVM_ENABLE_ZLIB=ON \\
   -DLLVM_ENABLE_ZSTD=OFF \\
   -DLLVM_ENABLE_Z3_SOLVER=OFF \\
   -DLLVM_ENABLE_DIA_SDK=OFF \\

diff --git a/bazel/toolchain/Dockerfile.sysroot b/bazel/toolchain/Dockerfile.sysroot
@@ -0,0 +1,37 @@
+FROM ubuntu:jammy AS pkgs
+
+ENV DEBIAN_FRONTEND=noninteractive
+SHELL ["/bin/bash", "-c"]
+
+RUN apt-get update && apt-get install -y \
+    libc6 \
+    libc6-dev \
+    libc6-dbg \
+    libgcc-12-dev \
+    valgrind \
+    xfslibs-dev \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Copy required glibc files
+RUN mkdir -p /output/pkgs/{usr/,}lib/$(uname -m)-linux-gnu /output/pkgs/usr/include /output/pkgs/lib64 \
+    && cp -r /usr/lib/*-linux-gnu/ld-linux-*.so* /output/pkgs/lib64/ \
+    && cp -r /usr/lib/*-linux-gnu/libc.{a,so}* /output/pkgs/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/*-linux-gnu/libc_nonshared.a* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/*-linux-gnu/libm.{a,so}* /output/pkgs/lib/$(uname -m)-linux-gnu/ \
+    && (if test $(uname -m) = "x86_64"; then cp -r /usr/lib/x86_64-linux-gnu/libmvec.{a,so}* /output/pkgs/lib/x86_64-linux-gnu/; fi) \
+    && cp -r /usr/lib/*-linux-gnu/libdl.{a,so}* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/*-linux-gnu/librt.{a,so}* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/*-linux-gnu/libutil.{a,so}* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/*-linux-gnu/libpthread.{a,so}* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/*-linux-gnu/libgcc_s.so* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/*-linux-gnu/*crt*.o /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/*-linux-gnu/libatomic.so* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/gcc/*-linux-gnu/12/*.o /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/lib/gcc/*-linux-gnu/12/libgcc*.{a,so}* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
+    && cp -r /usr/include/* /output/pkgs/usr/include/
+
+FROM scratch
+
+COPY --from=pkgs /output/pkgs /
+
+
diff --git a/bazel/toolchain/README.md b/bazel/toolchain/README.md
@@ -28,3 +28,21 @@ docker buildx build --platform=linux/arm64 --build-arg "LLVM_VERSION=17.0.6" --f
 
 By default we build with PGO+LTO, but if PGO is causing issues (like on AArch64), we can choose a different build (resulting
 in a slower compiler) by adding the flag `--target=lto`. The current default target is `--target=pgo`
+
+
+## Sysroot
+
+To make builds more hermetic we build with a sysroot from an older linux distro. These sysroots are crafted by creating a docker image with
+the correct packages in it, then extracting out the exact set of headers and shared libraries that are needed.
+
+To build an `x86_64` sysroot on an `x86_64` machine the following command can be used
+
+```
+OUTPUT_FILE="sysroot-ubuntu-22.04-x86_64-$(date --rfc-3339=date -u).tar.gz" docker build --file Dockerfile.sysroot --output type=tar,dest=- . | gzip > "$OUTPUT_FILE"
+```
+
+Building for `arm64` can be done from an `x86_64` host with the following command
+
+```
+OUTPUT_FILE="sysroot-ubuntu-22.04-aarch64-$(date --rfc-3339=date -u).tar.gz" docker buildx build --platform=linux/arm64 --file Dockerfile.sysroot --output type=tar,dest=- . | gzip > "$OUTPUT_FILE"
+```