diff --git a/.circleci/config.yml b/.circleci/config.yml index 019ed770c..629bb5c0c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ version: "2.1" executors: cross-builder: docker: - - image: quay.io/influxdb/builder:kapacitor-20240531 + - image: quay.io/influxdb/builder:kapacitor-20240611 resource_class: large linux-amd64: machine: diff --git a/Dockerfile_build_ubuntu64 b/Dockerfile_build_ubuntu64 index f405da33c..a2e84d2f2 100644 --- a/Dockerfile_build_ubuntu64 +++ b/Dockerfile_build_ubuntu64 @@ -1,4 +1,4 @@ -FROM quay.io/influxdb/builder:kapacitor-20240531 +FROM quay.io/influxdb/builder:kapacitor-20240611 # This dockerfile is capabable of performing all # build/test/package/deploy actions needed for Kapacitor. diff --git a/builder/Dockerfile_build b/builder/Dockerfile_build index 651a70049..f0d6924c3 100644 --- a/builder/Dockerfile_build +++ b/builder/Dockerfile_build @@ -8,25 +8,31 @@ MAINTAINER support@influxdb.com ARG DEBIAN_FRONTEND=noninteractive RUN apt-get -qq update && apt-get -qq install -y \ - software-properties-common \ - unzip \ + autoconf \ + automake \ + clang \ + gcc \ + gcc-aarch64-linux-gnu \ + git \ + libtool \ + llvm-dev \ + lzma-dev \ + mingw-w64 \ mercurial \ make \ - ruby \ - ruby-dev \ - rpm \ - zip \ + pkg-config \ python \ python-setuptools \ python3 \ python3-setuptools \ python3-boto \ - autoconf \ - automake \ - libtool \ + ruby \ + ruby-dev \ + rpm \ + software-properties-common \ + unzip \ wget \ - git \ - pkg-config + zip RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 @@ -54,7 +60,7 @@ ENV PATH $GOPATH/bin:$PATH # # src: https://github.com/influxdata/edge/blob/4677c285014ac27727e5a1ae9bf2c1633afc6ea6/dockerfiles/cross-builder/install-rust.sh#L7 # -ENV RUST_LATEST_VERSION=1.78.0 +ENV RUST_LATEST_VERSION=1.63.0 # For security, we specify a particular rustup version and a SHA256 hash, computed # ourselves and hardcoded here. When updating `RUSTUP_LATEST_VERSION`: # 1. Download the new rustup script from https://github.com/rust-lang/rustup/releases. @@ -78,6 +84,58 @@ RUN . $HOME/.cargo/env && rustup target add \ x86_64-unknown-linux-musl ENV PATH=/root/.cargo/bin:${PATH} +# +# src: https://github.com/influxdata/edge/blob/4677c285014ac27727e5a1ae9bf2c1633afc6ea6/dockerfiles/cross-builder/Dockerfile#L53 +# +COPY fs/ / + +RUN git clone git://git.musl-libc.org/musl /tmp/musl \ + --branch v1.2.3 --depth 1 + +# Enable "-static-pie" Compiler Flag +RUN patch /tmp/musl/tools/musl-gcc.specs.sh \ + /usr/local/src/musl-gcc/musl-gcc.specs.sh.patch + +# +# Build MUSL AMD64 Compiler +# +# src: https://github.com/influxdata/edge/blob/4677c285014ac27727e5a1ae9bf2c1633afc6ea6/dockerfiles/cross-builder/Dockerfile#L62 +# +RUN cd /tmp/musl && \ + export CC=gcc && \ + export AR=ar && \ + export RANLIB=ranlib && \ + ./configure \ + --enable-wrapper=gcc \ + --prefix=/musl/x86_64 \ + --target=x86_64-linux-musl && \ + make -j"$(nprocs)" install && \ + make clean +# BUILD MUSL ARM64 Compiler +RUN cd /tmp/musl && \ + export CC=aarch64-linux-gnu-gcc && \ + export AR=aarch64-linux-gnu-ar && \ + export RANLIB=aarch64-linux-gnu-ranlib && \ + ./configure \ + --enable-wrapper=gcc \ + --prefix=/musl/aarch64 \ + --target=aarch64-linux-musl && \ + make -j"$(nprocs)" install && \ + make clean + +# +# Install osxcross +# +# src: https://github.com/influxdata/edge/blob/cb1343dd74ecba8ec07fe810195530a0b9055aa9/dockerfiles/cross-builder/Dockerfile#L85 +# +ENV OSXCROSS_VERSION=5771a847950abefed9a37e2d16ee10e0dd90c641 +ENV OSXCROSS_BUILD_TIME=20220412215428 +RUN OSXCROSS_ARCHIVE=osxcross-${OSXCROSS_VERSION}-${OSXCROSS_BUILD_TIME}.tar.gz && \ + wget https://edge-xcc-archives.s3-us-west-2.amazonaws.com/${OSXCROSS_ARCHIVE} && \ + tar xzf ${OSXCROSS_ARCHIVE} -C /usr/local && \ + rm ${OSXCROSS_ARCHIVE} +ENV PATH=/usr/local/osxcross/target/bin:${PATH} + # # Install protobuf3 runtime and protoc binary # diff --git a/builder/README.md b/builder/README.md index 190435be7..c19611935 100644 --- a/builder/README.md +++ b/builder/README.md @@ -1,15 +1,21 @@ # Builds -Our CI/CD pipelines utilize a Docker build image configured with support for GoLang, Rust, and Protobuf. The `circle.yml` file references this Docker container to handle building, testing, and creating release packages. +Our CI/CD pipelines utilize a Docker build image configured with support for GoLang, Rust, MUSL, OSXCross and Protobuf. The `circle.yml` file references this Docker container to handle building, testing, and creating release packages. ## Custom Builder The necessity for a custom builder arises from compatibility issues between the `protobuf` library and Chronograf's Python UDFs. The `cross-builder` was updated to `protobuf` version `26.1` in [PR #669](https://github.com/influxdata/edge/pull/669), introducing breaking changes in the Python protobuf library. Specifically, [protobuf 5.26.1 on PyPI](https://pypi.org/project/protobuf/5.26.1/) does not support Python 2. Consequently, using the newest `cross-builder` would result in the loss of Python v2 support in UDFs. +:warning: **Note:** The custom builder depends on the `MUSL` compiler. In the current state (`2024-06-11`) the `MUSL` compiler requires Intel processor to build. This means that the custom builder is not able to __build__ on Apple Silicon hardware. + ## Updating Component Versions To update component versions like GoLang, Rust, and Protobuf, modifications must be made in `Dockerfile_build`. After updates, a new Docker image needs to be built, published, and then utilized in CI. +### Rust + +The Rust version is defined in the `Dockerfile_build` file. The Rust version should be same as the used version for `flux` library. + ### Step 1: Authenticate with Quay.io ```sh @@ -30,3 +36,11 @@ cd $KAPACITOR_REPOSITORY_ROOT/builder 1. Update the `cross-builder` tag in `.circleci/config.yml` to the new version. 2. Update the `quay.io/influxdb/builder` tag in `Dockerfile_build_ubuntu64` to reflect the new version. + +### Step 4: Test the New Builder + +To test new deployed builder you should run the following command in the root directory to build the `kapacitor`: + +```sh +./build.sh --debug --clean --generate --package --package-udfs --platform=all --arch=all --checksum +``` diff --git a/builder/fs/root/.cargo/config b/builder/fs/root/.cargo/config new file mode 100644 index 000000000..3e261e597 --- /dev/null +++ b/builder/fs/root/.cargo/config @@ -0,0 +1,16 @@ +# Eventually rust might support cross compilation out of the box, but until then +# we have to tell it which external linker to use: +# https://rust-lang.github.io/rustup/cross-compilation.html +[target.x86_64-apple-darwin] +linker = "x86_64-apple-darwin18-clang" +ar = "x86_64-apple-darwin18-ar" + +[target.x86_64-pc-windows-gnu] +linker = "x86_64-w64-mingw32-gcc" + +[target.aarch64-unknown-linux-musl] +linker = "/musl/aarch64/bin/musl-gcc" + +[target.x86_64-unknown-linux-musl] +linker = "/musl/x86_64/bin/musl-gcc" + diff --git a/builder/fs/usr/local/bin/build-version.sh b/builder/fs/usr/local/bin/build-version.sh new file mode 100755 index 000000000..a873cca37 --- /dev/null +++ b/builder/fs/usr/local/bin/build-version.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -eo pipefail + +function main () { + if [[ $# != 1 ]]; then + >&2 echo Usage: $0 '' + >&2 echo "Valid build types are 'release', 'nightly', and 'snapshot'" + exit 1 + fi + local -r build_type=$1 + + local version + case "$build_type" in + release) + if [ -n "$CIRCLE_TAG" ]; then + version="$CIRCLE_TAG" + else + version=$(git describe --tags --abbrev=0 --exact-match) + fi + ;; + nightly) + version=$(git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0)+nightly.$(date +%Y.%m.%d) + ;; + snapshot) + version=$(git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0)+SNAPSHOT.$(git rev-parse --short HEAD) + ;; + *) + >&2 echo "Error: unknown build type '$build_type'" + >&2 echo "Valid build types are 'release', 'nightly', and 'snapshot'" + ;; + esac + if [ -z "$version" ]; then + >&2 echo "Error: couldn't compute version for build type '$build_type'" + exit 1 + fi + + echo "$version" +} + +main ${@} diff --git a/builder/fs/usr/local/bin/pkg-config.sh b/builder/fs/usr/local/bin/pkg-config.sh new file mode 100755 index 000000000..20acc5e42 --- /dev/null +++ b/builder/fs/usr/local/bin/pkg-config.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +tmpdir=$(mktemp -d) +trap "{ rm -rf ${tmpdir}; }" EXIT + +# "go build" can be noisy, and when Go invokes pkg-config (by calling this script) it will merge stdout and stderr. +# Discard any output unless "go build" terminates with an error. +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${tmpdir}/pkg-config github.com/influxdata/pkg-config &> ${tmpdir}/go_build_output +if [ "$?" -ne 0 ]; then + cat ${tmpdir}/go_build_output 1>&2 + exit 1 +fi + +${tmpdir}/pkg-config "$@" diff --git a/builder/fs/usr/local/bin/xcc b/builder/fs/usr/local/bin/xcc new file mode 100755 index 000000000..b795624d1 --- /dev/null +++ b/builder/fs/usr/local/bin/xcc @@ -0,0 +1,32 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +# ${1} -> platform +# ${2} -> architecture + +case "${1:-}" in + linux) + case "${2:-}" in + x86_64) + printf '/musl/x86_64/bin/musl-gcc\n' + exit 0 + ;; + aarch64) + printf '/musl/aarch64/bin/musl-gcc\n' + exit 0 + ;; + esac + ;; + darwin) + printf 'x86_64-apple-darwin18-clang\n' + exit 0 + ;; + windows) + printf 'x86_64-w64-mingw32-gcc\n' + exit 0 + ;; +esac + +exit 1 diff --git a/builder/fs/usr/local/src/musl-gcc/musl-gcc.specs.sh.patch b/builder/fs/usr/local/src/musl-gcc/musl-gcc.specs.sh.patch new file mode 100644 index 000000000..21ab0537f --- /dev/null +++ b/builder/fs/usr/local/src/musl-gcc/musl-gcc.specs.sh.patch @@ -0,0 +1,21 @@ +This patch was sourced from NVIDIA: +https://github.com/NVIDIA/enroot/blob/07b2288997a89f0278d049ece3ea3d4323a71abc/deps/musl.patch + +index 30492574..7206cb25 100644 +--- a/tools/musl-gcc.specs.sh ++++ b/tools/musl-gcc.specs.sh +@@ -17,13 +17,13 @@ cat <