diff --git a/bash/common.sh b/bash/common.sh index 01910104..70405541 100644 --- a/bash/common.sh +++ b/bash/common.sh @@ -1,16 +1,23 @@ #!/usr/bin/env bash # logger utility, output ANSI-colored messages to stderr; first argument is level (debug/info/warn/error), all other arguments are the message. -declare -A log_colors=(["debug"]="0;36" ["info"]="0;32" ["warn"]="0;33" ["error"]="0;31") -declare -A log_emoji=(["debug"]="🐛" ["info"]="📗" ["warn"]="🚧" ["error"]="🚨") +declare -A log_colors=(["debug"]="0;36" ["info"]="0;32" ["notice"]="1;32" ["warn"]="1;33" ["error"]="1;31") +declare -A log_emoji=(["debug"]="🐛" ["info"]="🌿" ["notice"]="🌱" ["warn"]="🚸" ["error"]="🚨") +declare -A log_gha_levels=(["notice"]="notice" ["warn"]="warning" ["error"]="error") function log() { declare level="${1}" shift [[ "${level}" == "debug" && "${DEBUG}" != "yes" ]] && return # Skip debugs unless DEBUG=yes is set in the environment - declare color="${log_colors[${level}]}" + # If running on GitHub Actions, and level exists in log_gha_levels... + if [[ -n "${GITHUB_ACTIONS}" && -n "${log_gha_levels[${level}]}" ]]; then + echo "::${log_gha_levels[${level}]} ::${*}" >&2 + fi + # Normal output + declare color="\033[${log_colors[${level}]}m" declare emoji="${log_emoji[${level}]}" + declare ansi_reset="\033[0m" level=$(printf "%-5s" "${level}") # pad to 5 characters before printing - echo -e "${emoji} \033[${color}m${SECONDS}: [${level}] $*\033[0m" >&2 + echo -e "${emoji} ${ansi_reset}[${color}${level}${ansi_reset}] ${color}${*}${ansi_reset}" >&2 } function install_dependencies() { diff --git a/bash/json-matrix.sh b/bash/json-matrix.sh index 36433060..46488fdd 100644 --- a/bash/json-matrix.sh +++ b/bash/json-matrix.sh @@ -73,7 +73,7 @@ function prepare_json_matrix() { if [[ "${matrix_type}" == "KERNEL" ]]; then # special case for kernel builds, if USE_KERNEL_ID is set, skip this kernel if [[ -n "${kernel_info[USE_KERNEL_ID]}" ]]; then - log warn "Skipping build of kernel '${kernel}' due to it having USE_KERNEL_ID set to '${kernel_info[USE_KERNEL_ID]}'" + log info "Skipping build of kernel '${kernel}' due to it having USE_KERNEL_ID set to '${kernel_info[USE_KERNEL_ID]}'" continue fi fi diff --git a/bash/kernel/kernel_armbian.sh b/bash/kernel/kernel_armbian.sh index bc811950..5fc55209 100644 --- a/bash/kernel/kernel_armbian.sh +++ b/bash/kernel/kernel_armbian.sh @@ -45,8 +45,15 @@ function calculate_kernel_version_armbian() { declare -g ARMBIAN_KERNEL_DOCKERFILE="kernel/Dockerfile.autogen.armbian.${inventory_id}" - declare oras_version="1.2.0-beta.1" # @TODO bump this once it's released; yes it's much better than 1.1.x's - declare oras_down_url="https://github.com/oras-project/oras/releases/download/v${oras_version}/oras_${oras_version}_linux_amd64.tar.gz" + declare oras_version="1.2.0-rc.1" # @TODO bump this once it's released; yes it's much better than 1.1.x's + # determine the arch to download from current arch + declare oras_arch="unknown" + case "$(uname -m)" in + "x86_64") oras_arch="amd64" ;; + "aarch64") oras_arch="arm64" ;; + *) log error "ERROR: ARCH $(uname -m) not supported by ORAS? check https://github.com/oras-project/oras/releases" && exit 1 ;; + esac + declare oras_down_url="https://github.com/oras-project/oras/releases/download/v${oras_version}/oras_${oras_version}_linux_${oras_arch}.tar.gz" # Lets create a Dockerfile that will be used to obtain the artifacts needed, using ORAS binary echo "Creating Dockerfile '${ARMBIAN_KERNEL_DOCKERFILE}'... " @@ -83,8 +90,16 @@ function calculate_kernel_version_armbian() { # Get the kernel image... RUN cp -v boot/vmlinuz* /armbian/output/kernel - # Create a tarball with the modules in lib - RUN tar -cvf /armbian/output/kernel.tar lib + # Create a tarball with the modules in lib. + # Important: this tarball needs to have permissions for the root directory included! Otherwise linuxkit rootfs will have the wrong permissions on / (root) + WORKDIR /armbian/modules_only + RUN mv /armbian/image/lib /armbian/modules_only/ + RUN echo "Before cleaning: " && du -h -d 10 -x . | sort -h | tail -n 20 + # Trim the kernel modules to save space; hopefully your required hardware is not included here + RUN rm -rfv ./lib/modules/*/kernel/drivers/net/wireless ./lib/modules/*/kernel/sound ./lib/modules/*/kernel/drivers/media + RUN rm -rfv ./lib/modules/*/kernel/drivers/infiniband + RUN echo "After cleaning: " && du -h -d 10 -x . | sort -h | tail -n 20 + RUN tar -cf /armbian/output/kernel.tar . # Create a tarball with the dtbs in usr/lib/linux-image-* RUN { cd usr/lib/linux-image-* || { echo "No DTBS for this arch, empty tar..." && mkdir -p usr/lib/linux-image-no-dtbs && cd usr/lib/linux-image-* ; } ; } && pwd && du -h -d 1 . && tar -czvf /armbian/output/dtbs.tar.gz . && ls -lah /armbian/output/dtbs.tar.gz diff --git a/bash/linuxkit.sh b/bash/linuxkit.sh index 32a95ef3..94edfaf7 100644 --- a/bash/linuxkit.sh +++ b/bash/linuxkit.sh @@ -84,15 +84,31 @@ function linuxkit_build() { declare -a lk_args=( "--docker" "--arch" "${kernel_info['DOCKER_ARCH']}" - "--format" "kernel+initrd" "--name" "hook" "--cache" "${lk_cache_dir}" "--dir" "${lk_output_dir}" "hook.${inventory_id}.yaml" # the linuxkit configuration file ) + if [[ "${OUTPUT_TARBALL_FILELIST:-"no"}" == "yes" ]]; then + log info "OUTPUT_TARBALL_FILELIST=yes; Building Hook (tar/filelist) with kernel ${inventory_id} using linuxkit: ${lk_args[*]}" + "${linuxkit_bin}" build "--format" "tar" "${lk_args[@]}" + fi + log info "Building Hook with kernel ${inventory_id} using linuxkit: ${lk_args[*]}" - "${linuxkit_bin}" build "${lk_args[@]}" + "${linuxkit_bin}" build "--format" "kernel+initrd" "${lk_args[@]}" + + declare initramfs_path="${lk_output_dir}/hook-initrd.img" + # initramfs_path is a gzipped file. obtain the uncompressed byte size, without decompressing it + declare -i initramfs_size_bytes=0 + initramfs_size_bytes=$(gzip -l "${initramfs_path}" | tail -n 1 | awk '{print $2}') + log info "Uncompressed initramfs size in bytes: ${initramfs_size_bytes}" + # If the size is larger than 900mb, it is unlikely to boot on a 2gb RAM machine. Warn. + if [[ "${initramfs_size_bytes}" -gt 943718400 ]]; then + log warn "${inventory_id}: Uncompressed initramfs size (${initramfs_size_bytes} bytes) is larger than 900mb; it may not boot on a 2gb RAM machine." + else + log notice "${inventory_id}: Uncompressed initramfs size (${initramfs_size_bytes} bytes) is smaller than 900mb." + fi if [[ "${LK_RUN}" == "qemu" ]]; then linuxkit_run_qemu @@ -137,6 +153,13 @@ function linuxkit_build() { rm -rf "${dtbs_tmp_dir}" rm "${lk_output_dir}/dtbs-${OUTPUT_ID}.tar.gz" + if [[ "${OUTPUT_TARBALL_FILELIST:-"no"}" == "yes" ]]; then + log info "OUTPUT_TARBALL_FILELIST=yes; including tar and filelist in output." + mv -v "${lk_output_dir}/hook.tar" "out/hook/hook_rootfs_${OUTPUT_ID}.tar" + tar --list -vf "out/hook/hook_rootfs_${OUTPUT_ID}.tar" > "out/hook/hook_rootfs_${OUTPUT_ID}.filelist" + fi + + # finally clean up the hook-specific out dir rm -rf "${lk_output_dir}" # tar the files into out/hook.tar in such a way that vmlinuz and initramfs are at the root of the tar; pigz it diff --git a/images/hook-bootkit/Dockerfile b/images/hook-bootkit/Dockerfile index 45fd8791..6dc81241 100644 --- a/images/hook-bootkit/Dockerfile +++ b/images/hook-bootkit/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.21-alpine as dev COPY . /src/ WORKDIR /src RUN go mod download -RUN CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "-static"' -o /bootkit +RUN CGO_ENABLED=0 go build -a -ldflags '-s -w -extldflags "-static"' -o /bootkit FROM alpine COPY --from=dev /bootkit . diff --git a/images/hook-bootkit/hook-bootkit b/images/hook-bootkit/hook-bootkit deleted file mode 100755 index 11a107a1..00000000 Binary files a/images/hook-bootkit/hook-bootkit and /dev/null differ diff --git a/images/hook-containerd/Dockerfile b/images/hook-containerd/Dockerfile index 755514e6..adfa92bd 100644 --- a/images/hook-containerd/Dockerfile +++ b/images/hook-containerd/Dockerfile @@ -18,7 +18,7 @@ RUN mkdir -p $GOPATH/src/github.com/containerd && \ git checkout $CONTAINERD_COMMIT RUN apk add --no-cache btrfs-progs-dev gcc libc-dev linux-headers make libseccomp-dev WORKDIR $GOPATH/src/github.com/containerd/containerd -RUN make binaries EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' BUILDTAGS="static_build no_devmapper" +RUN make binaries EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-w -s -extldflags "-fno-PIC -static"' BUILDTAGS="static_build no_devmapper" # install nerdctl RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then ARCHITECTURE=amd64; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then ARCHITECTURE=arm64; else ARCHITECTURE=amd64; fi \ diff --git a/images/hook-docker/Dockerfile b/images/hook-docker/Dockerfile index 778db306..3aad7bd1 100644 --- a/images/hook-docker/Dockerfile +++ b/images/hook-docker/Dockerfile @@ -1,10 +1,16 @@ FROM golang:1.20-alpine as dev COPY . /src/ WORKDIR /src -RUN CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "-static"' -o /hook-docker +RUN CGO_ENABLED=0 go build -a -ldflags '-s -w -extldflags "-static"' -o /hook-docker FROM docker:26.1.0-dind RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories -RUN apk update; apk add kexec-tools +RUN apk update && apk add kexec-tools binutils && rm -rf /var/cache/apk/* +# Won't use docker-buildx nor docker-compose +RUN rm -rf /usr/local/libexec/docker/cli-plugins +# Strip some large binaries +RUN strip /usr/local/bin/docker /usr/local/bin/dockerd /usr/local/bin/docker-proxy /usr/local/bin/runc +# Purge binutils package after stripping +RUN apk del binutils COPY --from=dev /hook-docker . ENTRYPOINT ["/hook-docker"] diff --git a/images/hook-mdev/Dockerfile b/images/hook-mdev/Dockerfile index bd3ae97c..3415ff33 100644 --- a/images/hook-mdev/Dockerfile +++ b/images/hook-mdev/Dockerfile @@ -2,7 +2,7 @@ FROM alpine USER root:root -RUN apk add --no-cache mdev-conf +RUN apk add --no-cache mdev-conf && rm -rf /var/cache/apk/* CMD ["mdev", "-v", "-df"] diff --git a/images/hook-runc/Dockerfile b/images/hook-runc/Dockerfile index 8101a91c..a5fb86b8 100644 --- a/images/hook-runc/Dockerfile +++ b/images/hook-runc/Dockerfile @@ -19,7 +19,7 @@ RUN mkdir -p $GOPATH/src/github.com/opencontainers && \ git clone https://github.com/opencontainers/runc.git WORKDIR $GOPATH/src/github.com/opencontainers/runc RUN git checkout $RUNC_COMMIT -RUN make static BUILDTAGS="seccomp" EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS="-extldflags \\\"-fno-PIC -static\\\"" +RUN make static BUILDTAGS="seccomp" EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS="-s -w -extldflags \\\"-fno-PIC -static\\\"" RUN cp runc /usr/bin/ RUN mkdir -p /etc/init.d && ln -s /usr/bin/service /etc/init.d/010-onboot diff --git a/linuxkit-templates/hook.template.yaml b/linuxkit-templates/hook.template.yaml index 8d74ba83..cc8aaf37 100644 --- a/linuxkit-templates/hook.template.yaml +++ b/linuxkit-templates/hook.template.yaml @@ -134,6 +134,16 @@ services: major: 204 minor: 65 mode: "0666" + - path: "/dev/ttyAML0" + type: c + major: 243 + minor: 0 + mode: "0666" + - path: "/dev/ttyAML1" + type: c + major: 243 + minor: 1 + mode: "0666" - name: hook-docker image: "${HOOK_CONTAINER_DOCKER_IMAGE}" @@ -266,6 +276,8 @@ files: ttyS1 ttyAMA0 ttyAMA1 + ttyAML0 + ttyAML1 ttyUSB0 ttyUSB1 ttyUSB2