chore: add missing deps for crossbuilder (#2823)

.circleci/config.yml

@@ -4,7 +4,7 @@ version:  "2.1"
 executors:
   cross-builder:
     docker:
-      - image: quay.io/influxdb/builder:kapacitor-20240531
+      - image: quay.io/influxdb/builder:kapacitor-20240611
     resource_class: large
   linux-amd64:
     machine:

Dockerfile_build_ubuntu64

@@ -1,4 +1,4 @@
-FROM quay.io/influxdb/builder:kapacitor-20240531
+FROM quay.io/influxdb/builder:kapacitor-20240611
 
 # This dockerfile is capabable of performing all
 # build/test/package/deploy actions needed for Kapacitor.

builder/Dockerfile_build

@@ -8,25 +8,31 @@ MAINTAINER [email protected]
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get -qq update && apt-get -qq install -y \
-    software-properties-common \
-    unzip \
+    autoconf \
+    automake \
+    clang \
+    gcc \
+    gcc-aarch64-linux-gnu \
+    git \
+    libtool \
+    llvm-dev \
+    lzma-dev \
+    mingw-w64 \
     mercurial \
     make \
-    ruby \
-    ruby-dev \
-    rpm \
-    zip \
+    pkg-config \
     python \
     python-setuptools \
     python3 \
     python3-setuptools \
     python3-boto \
-    autoconf \
-    automake \
-    libtool \
+    ruby \
+    ruby-dev \
+    rpm \
+    software-properties-common \
+    unzip \
     wget \
-    git \
-    pkg-config
+    zip
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 
@@ -54,7 +60,7 @@ ENV PATH $GOPATH/bin:$PATH
 #
 # src: https://github.com/influxdata/edge/blob/4677c285014ac27727e5a1ae9bf2c1633afc6ea6/dockerfiles/cross-builder/install-rust.sh#L7
 #
-ENV RUST_LATEST_VERSION=1.78.0
+ENV RUST_LATEST_VERSION=1.63.0
 # For security, we specify a particular rustup version and a SHA256 hash, computed
 # ourselves and hardcoded here. When updating `RUSTUP_LATEST_VERSION`:
 #   1. Download the new rustup script from https://github.com/rust-lang/rustup/releases.
@@ -78,6 +84,58 @@ RUN . $HOME/.cargo/env && rustup target add \
         x86_64-unknown-linux-musl
 ENV PATH=/root/.cargo/bin:${PATH}
 
+#
+# src: https://github.com/influxdata/edge/blob/4677c285014ac27727e5a1ae9bf2c1633afc6ea6/dockerfiles/cross-builder/Dockerfile#L53
+#
+COPY fs/ /
+
+RUN git clone git://git.musl-libc.org/musl /tmp/musl \
+      --branch v1.2.3 --depth 1
+
+# Enable "-static-pie" Compiler Flag
+RUN patch /tmp/musl/tools/musl-gcc.specs.sh \
+          /usr/local/src/musl-gcc/musl-gcc.specs.sh.patch
+
+#
+# Build MUSL AMD64 Compiler
+#
+# src: https://github.com/influxdata/edge/blob/4677c285014ac27727e5a1ae9bf2c1633afc6ea6/dockerfiles/cross-builder/Dockerfile#L62
+#
+RUN cd /tmp/musl                 && \
+    export CC=gcc                && \
+    export AR=ar                 && \
+    export RANLIB=ranlib         && \
+    ./configure                     \
+      --enable-wrapper=gcc          \
+      --prefix=/musl/x86_64         \
+      --target=x86_64-linux-musl && \
+    make -j"$(nprocs)" install   && \
+    make clean
+# BUILD MUSL ARM64 Compiler
+RUN cd /tmp/musl                           && \
+    export CC=aarch64-linux-gnu-gcc        && \
+    export AR=aarch64-linux-gnu-ar         && \
+    export RANLIB=aarch64-linux-gnu-ranlib && \
+    ./configure                               \
+      --enable-wrapper=gcc                    \
+      --prefix=/musl/aarch64                  \
+      --target=aarch64-linux-musl          && \
+    make -j"$(nprocs)" install             && \
+    make clean
+
+#
+# Install osxcross
+#
+# src: https://github.com/influxdata/edge/blob/cb1343dd74ecba8ec07fe810195530a0b9055aa9/dockerfiles/cross-builder/Dockerfile#L85
+#
+ENV OSXCROSS_VERSION=5771a847950abefed9a37e2d16ee10e0dd90c641
+ENV OSXCROSS_BUILD_TIME=20220412215428
+RUN OSXCROSS_ARCHIVE=osxcross-${OSXCROSS_VERSION}-${OSXCROSS_BUILD_TIME}.tar.gz && \
+    wget https://edge-xcc-archives.s3-us-west-2.amazonaws.com/${OSXCROSS_ARCHIVE} && \
+    tar xzf ${OSXCROSS_ARCHIVE} -C /usr/local && \
+    rm ${OSXCROSS_ARCHIVE}
+ENV PATH=/usr/local/osxcross/target/bin:${PATH}
+
 #
 # Install protobuf3 runtime and protoc binary
 #

builder/README.md

@@ -1,15 +1,21 @@
 # Builds
 
-Our CI/CD pipelines utilize a Docker build image configured with support for GoLang, Rust, and Protobuf. The `circle.yml` file references this Docker container to handle building, testing, and creating release packages.
+Our CI/CD pipelines utilize a Docker build image configured with support for GoLang, Rust, MUSL, OSXCross and Protobuf. The `circle.yml` file references this Docker container to handle building, testing, and creating release packages.
 
 ## Custom Builder
 
 The necessity for a custom builder arises from compatibility issues between the `protobuf` library and Chronograf's Python UDFs. The `cross-builder` was updated to `protobuf` version `26.1` in [PR #669](https://github.com/influxdata/edge/pull/669), introducing breaking changes in the Python protobuf library. Specifically, [protobuf 5.26.1 on PyPI](https://pypi.org/project/protobuf/5.26.1/) does not support Python 2. Consequently, using the newest `cross-builder` would result in the loss of Python v2 support in UDFs.
 
+:warning: **Note:** The custom builder depends on the `MUSL` compiler. In the current state (`2024-06-11`) the `MUSL` compiler requires Intel processor to build. This means that the custom builder is not able to __build__ on Apple Silicon hardware.
+
 ## Updating Component Versions
 
 To update component versions like GoLang, Rust, and Protobuf, modifications must be made in `Dockerfile_build`. After updates, a new Docker image needs to be built, published, and then utilized in CI.
 
+### Rust
+
+The Rust version is defined in the `Dockerfile_build` file. The Rust version should be same as the used version for `flux` library.
+
 ### Step 1: Authenticate with Quay.io
 
 ```sh
@@ -30,3 +36,11 @@ cd $KAPACITOR_REPOSITORY_ROOT/builder
 
 1. Update the `cross-builder` tag in `.circleci/config.yml` to the new version.
 2. Update the `quay.io/influxdb/builder` tag in `Dockerfile_build_ubuntu64` to reflect the new version.
+
+### Step 4: Test the New Builder
+
+To test new deployed builder you should run the following command in the root directory to build the `kapacitor`:
+
+```sh
+./build.sh --debug --clean --generate --package --package-udfs --platform=all --arch=all --checksum
+```

builder/fs/root/.cargo/config

@@ -0,0 +1,16 @@
+# Eventually rust might support cross compilation out of the box, but until then
+# we have to tell it which external linker to use:
+# https://rust-lang.github.io/rustup/cross-compilation.html
+[target.x86_64-apple-darwin]
+linker = "x86_64-apple-darwin18-clang"
+ar     = "x86_64-apple-darwin18-ar"
+
+[target.x86_64-pc-windows-gnu]
+linker = "x86_64-w64-mingw32-gcc"
+
+[target.aarch64-unknown-linux-musl]
+linker = "/musl/aarch64/bin/musl-gcc"
+
+[target.x86_64-unknown-linux-musl]
+linker = "/musl/x86_64/bin/musl-gcc"
+

builder/fs/usr/local/bin/build-version.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+function main () {
+    if [[ $# != 1 ]]; then
+        >&2 echo Usage: $0 '<build-type>'
+        >&2 echo "Valid build types are 'release', 'nightly', and 'snapshot'"
+        exit 1
+    fi
+    local -r build_type=$1
+
+    local version
+    case "$build_type" in
+        release)
+            if [ -n "$CIRCLE_TAG" ]; then
+                version="$CIRCLE_TAG"
+            else
+                version=$(git describe --tags --abbrev=0 --exact-match)
+            fi
+            ;;
+        nightly)
+            version=$(git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0)+nightly.$(date +%Y.%m.%d)
+            ;;
+        snapshot)
+            version=$(git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0)+SNAPSHOT.$(git rev-parse --short HEAD)
+            ;;
+        *)
+            >&2 echo "Error: unknown build type '$build_type'"
+            >&2 echo "Valid build types are 'release', 'nightly', and 'snapshot'"
+            ;;
+    esac
+    if [ -z "$version" ]; then
+        >&2 echo "Error: couldn't compute version for build type '$build_type'"
+        exit 1
+    fi
+
+    echo "$version"
+}
+
+main ${@}

builder/fs/usr/local/bin/pkg-config.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+tmpdir=$(mktemp -d)
+trap "{ rm -rf ${tmpdir}; }" EXIT
+
+# "go build" can be noisy, and when Go invokes pkg-config (by calling this script) it will merge stdout and stderr.
+# Discard any output unless "go build" terminates with an error.
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${tmpdir}/pkg-config github.com/influxdata/pkg-config &> ${tmpdir}/go_build_output
+if [ "$?" -ne 0 ]; then
+    cat ${tmpdir}/go_build_output 1>&2
+    exit 1
+fi
+
+${tmpdir}/pkg-config "$@"

builder/fs/usr/local/bin/xcc

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -o errexit \
+    -o nounset \
+    -o pipefail
+
+# ${1} -> platform
+# ${2} -> architecture
+
+case "${1:-}" in
+  linux)
+    case "${2:-}" in
+      x86_64)
+        printf '/musl/x86_64/bin/musl-gcc\n'
+        exit 0
+      ;;
+      aarch64)
+        printf '/musl/aarch64/bin/musl-gcc\n'
+        exit 0
+      ;;
+    esac
+  ;;
+  darwin)
+    printf 'x86_64-apple-darwin18-clang\n'
+    exit 0
+  ;;
+  windows)
+    printf 'x86_64-w64-mingw32-gcc\n'
+    exit 0
+  ;;
+esac
+
+exit 1

builder/fs/usr/local/src/musl-gcc/musl-gcc.specs.sh.patch

@@ -0,0 +1,21 @@
+This patch was sourced from NVIDIA:
+https://github.com/NVIDIA/enroot/blob/07b2288997a89f0278d049ece3ea3d4323a71abc/deps/musl.patch
+
+index 30492574..7206cb25 100644
+--- a/tools/musl-gcc.specs.sh
++++ b/tools/musl-gcc.specs.sh
+@@ -17,13 +17,13 @@ cat <<EOF
+ libgcc.a%s %:if-exists(libgcc_eh.a%s)
+
+ *startfile:
+-%{!shared: $libdir/Scrt1.o} $libdir/crti.o crtbeginS.o%s
++%{static-pie: $libdir/rcrt1.o} %{!static-pie: %{!shared: $libdir/Scrt1.o}} $libdir/crti.o crtbeginS.o%s
+
+ *endfile:
+ crtendS.o%s $libdir/crtn.o
+
+ *link:
+--dynamic-linker $ldso -nostdlib %{shared:-shared} %{static:-static} %{rdynamic:-export-dynamic}
++%{static-pie:-no-dynamic-linker -static -pie} %{!static-pie:-dynamic-linker $ldso} -nostdlib %{shared:-shared} %{static:-static} %{rdynamic:-export-dynamic}
+
+ *esp_link: