From c8bd4913ef28fcfa617163ef3c86dfcdd84e13f7 Mon Sep 17 00:00:00 2001 From: Vladimir Sokolovsky Date: Mon, 10 Jul 2023 17:14:45 +0000 Subject: [PATCH] Anolis8.6 DOCA_2.2.0_BSP_4.2.0 Beta Signed-off-by: Vladimir Sokolovsky --- anolis/8.6/Dockerfile | 75 +++++++ anolis/8.6/bfb-build | 66 ++++++ anolis/8.6/create_bfb | 339 ++++++++++++++++++++++++++++ anolis/8.6/install.sh | 500 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 980 insertions(+) create mode 100644 anolis/8.6/Dockerfile create mode 100755 anolis/8.6/bfb-build create mode 100755 anolis/8.6/create_bfb create mode 100755 anolis/8.6/install.sh diff --git a/anolis/8.6/Dockerfile b/anolis/8.6/Dockerfile new file mode 100644 index 0000000..5ca0607 --- /dev/null +++ b/anolis/8.6/Dockerfile @@ -0,0 +1,75 @@ +from --platform=linux/arm64 openanolis/anolisos:8.6 +ADD qemu-aarch64-static /usr/bin/ + +ARG BASE_URL +ARG DOCA_VERSION +ARG BSP_VERSION +ARG DISTRO +ARG DISTRO_VERSION +WORKDIR /root/workspace +ADD install.sh . +ADD create_bfb . +ADD bootimages bootimages/ + +ENV RUN_FW_UPDATER=no + +# Set releasever to 8.6 +RUN echo 8.6 > /etc/dnf/vars/dockerreleasever +RUN sed -i -e "s/releasever/dockerreleasever/g" /etc/yum.repos.d/*repo + +RUN dnf update -y +RUN dnf install -y epel-release + +RUN echo -e "[Experimental] \n\ +name=AnolisOS-8.6 - Experimental \n\ +baseurl=http://mirrors.openanolis.cn/anolis/\$dockerreleasever/Experimental/\$basearch/os \n\ +enabled=1 \n\ +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-ANOLIS \n\ +gpgcheck=1" > /etc/yum.repos.d/AnolisOS-Experimental.repo + +RUN echo -e "[kubernetes] \n\ +name=Kubernetes \n\ +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-aarch64 \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg \n\ +exclude=kubelet kubeadm kubectl kubernetes-cni" > /etc/yum.repos.d/kubernetes.repo + +RUN dnf install -y --disableexcludes=kubernetes \ + rpm-build rpm-sign automake meson cmake gcc-c++ \ + bc bison flex pesign rsync patchutils git annobin intltool \ + groff kernel-rpm-macros libtool desktop-file-utils jq wget \ + systemd lm_sensors-sensord openssh-server acpid irqbalance unbound rasdaemon \ + cryptsetup lvm2 device-mapper-persistent-data ltrace lsof unzip sysstat nvme-cli \ + usbutils kexec-tools nfs-utils dracut-tools dracut-network dracut-squash iproute-tc sudo parted passwd \ + network-scripts NetworkManager NetworkManager-ovs NetworkManager-config-server tcpdump \ + libarchive libreswan rsyslog system-lsb-core time vim numactl libiscsi \ + perl-Fedora-VSP perl-generators edac-util dmidecode \ + glib2-devel python3-devel elfutils-devel binutils-devel pciutils-devel openssl-devel \ + libnl3-devel selinux-policy-devel numactl-devel unbound-devel libpcap-devel tcl-devel \ + valgrind-devel iptables-devel libdb-devel libmnl-devel libcap-ng-devel systemd-devel \ + grubby grub2-efi-aa64 shim-aa64 efibootmgr re2c watchdog libmd libmd-devel libbsd libbsd-devel + +RUN dnf install -y --disableexcludes=kubernetes \ + kubelet-1.25.3 kubernetes-cni-1.1.1 cri-tools +RUN dnf install -y rpm-build rpm-sign automake meson cmake gcc-c++ + +RUN echo -e "[doca] \n\ +name=Nvidia DOCA repository \n\ +baseurl=$BASE_URL/doca/$DOCA_VERSION/$DISTRO$DISTRO_VERSION/aarch64/ \n\ +gpgcheck=0 \n\ +enabled=1" > /etc/yum.repos.d/doca.repo + +RUN dnf install -y kernel kernel-modules kernel-devel kernel-core kernel-modules-extra +RUN dnf install -y openssl-1.1.1q openssl-devel-1.1.1q openssl-libs-1.1.1q openssl-perl-1.1.1q openssl-static-1.1.1q + +RUN dnf install --nogpgcheck -y mstflint-4.16.1-2.2307026 +RUN dnf install --nogpgcheck -y doca-runtime doca-tools doca-sdk +RUN dnf install --nogpgcheck -y dpacc dpacc-extract dpa-compiler + +RUN /usr/sbin/update-pciids || true +RUN rpm -ihv --force bootimages/mlxbf-bootimages-*.aarch64.rpm || true + +RUN sed -i -e "s/signed/@IMAGE_TYPE@@CUSTOM_VERSION@/" -e "s/prod/@IMAGE_TYPE@@CUSTOM_VERSION@/" /etc/mlnx-release + +CMD ["/root/workspace/create_bfb", "-k", "5.10.134-13.1.an8.aarch64"] diff --git a/anolis/8.6/bfb-build b/anolis/8.6/bfb-build new file mode 100755 index 0000000..987515e --- /dev/null +++ b/anolis/8.6/bfb-build @@ -0,0 +1,66 @@ +#!/bin/bash -ex + +cd ${0%*/*} + +if [ ! -e Dockerfile ]; then + echo "ERROR: Dockerfile is missing." + exit 1 +fi + +if ! (which wget > /dev/null 2>&1); then + echo "wget is required to build BFB" + exit 1 +fi + +if ! (which docker > /dev/null 2>&1); then + echo "docker is required to build BFB" + exit 1 +fi + +DISTRO="anolis" +DISTRO_VERSION="8.6" +DOCA_VERSION="2.2.0-beta" +BSP_VERSION="4.2.0-12818" +IMAGE_TYPE=${IMAGE_TYPE:-"prod"} + +WDIR=/tmp/${DISTRO}${DISTRO_VERSION}.$$ +BASE_URL=${BASE_URL:-"https://linux.mellanox.com/public/repo"} + +mkdir -p $WDIR + +mkdir -p $WDIR/bootimages +wget -P $WDIR/bootimages -r --no-verbose --no-directories -l1 --no-parent -A 'mlxbf-bootimages*.aarch64.rpm' ${BASE_URL}/bluefield/${BSP_VERSION}/bootimages/${IMAGE_TYPE}/ + +cp Dockerfile \ + create_bfb \ + install.sh \ + ../../common/tools/qemu-aarch64-static \ + $WDIR + +cd $WDIR + +docker_image=bfb_runtime_${DISTRO,,}${DISTRO_VERSION,,} + +docker rm -f BlueField_OS_${DISTRO}_${DISTRO_VERSION} 2> /dev/null + +sed -i -e "s/@IMAGE_TYPE@/$IMAGE_TYPE/g;s/@CUSTOM_VERSION@/$CUSTOM_VERSION/g" Dockerfile + +docker build -t ${docker_image} \ + --build-arg BASE_URL=$BASE_URL \ + --build-arg DOCA_VERSION=$DOCA_VERSION \ + --build-arg BSP_VERSION=$BSP_VERSION \ + --build-arg DISTRO=$DISTRO \ + --build-arg DISTRO_VERSION=$DISTRO_VERSION \ + -f Dockerfile . + +docker run -t --rm --privileged -e container=docker \ + -v $PWD:/workspace \ + --name BlueField_OS_${DISTRO}_${DISTRO_VERSION} \ + --mount type=bind,source=/dev,target=/dev \ + --mount type=bind,source=/sys,target=/sys \ + --mount type=bind,source=/proc,target=/proc \ + ${docker_image} + +readlink -f *.bfb + +echo "Default root password is: anolis" diff --git a/anolis/8.6/create_bfb b/anolis/8.6/create_bfb new file mode 100755 index 0000000..cb24e9e --- /dev/null +++ b/anolis/8.6/create_bfb @@ -0,0 +1,339 @@ +#!/bin/bash + +# Copyright (c) 2020, Mellanox Technologies +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + +set -e + +LANG=C +BF_HOSTNAME="localhost" +SDIR="/root/workspace" +BFB="${BFB:-/lib/firmware/mellanox/boot/default.bfb}" +CAPSULE="${CAPSULE:-/lib/firmware/mellanox/boot/capsule/boot_update2.cap}" +verbose=0 +kdir=$(/bin/ls -1d /lib/modules/4.18* /lib/modules/4.19* /lib/modules/4.20* /lib/modules/5.* 2> /dev/null | head -1) +if [ -n "$kdir" ]; then + kernel=${kdir##*/} +else + kernel=$(/bin/ls -1 /lib/modules/ | head -1) +fi + +usage() { +cat << EOF +Usage: `basename $0` [ OPTIONS ] +OPTIONS: +-H, -hostname, --hostname Hostname for the SmartNIC. Default: $BF_HOSTNAME +-k, -kernel, --kernel Kernel version for the SmartNIC. Default: $kernel +-v, -verbose, --verbose Run script in verbose mode. Will print out each step of execution. +-h, -help, --help Display help + + +EOF +} + + +options=$(getopt -l "hostname:,kernel:,help,verbose" -o "H:k:hv" -a -- "$@") + +eval set -- "$options" + +while true +do + case $1 in + -h|--help) + usage + exit 0 + ;; + -H|--hostname) + shift + BF_HOSTNAME=$1 + ;; + -k|--kernel) + shift + kernel=$1 + ;; + -v|--verbose) + verbose=1 + set -xv + ;; + --) + shift + break;; + esac + shift +done + +bootctl_module="mlxbf-bootctl" + +modinfo -n -k $kernel "mlx-bootctl" 2> /dev/null && bootctl_module="mlx-bootctl" + +SCRIPTS_DIR=`dirname $0` + +WDIR=${WDIR:-/root/workspace/bfb} + +echo "Work directory: $WDIR" + +BFB_FILENAME="$(cat /etc/mlnx-release).bfb" + +PATH=$PATH:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/mellanox/scripts:/opt/mellanox/scripts/bin +mkbfb=`which mlx-mkbfb` + +if [ ! -x "${mkbfb}" ]; then + echo "- ERROR: mlx-mkbfb is required to build BFB image" + exit 1 +fi + +if [[ ! -n "${BFB}" || ! -e "${BFB}" ]]; then + echo "- ERROR: BFB was not found" + exit 1 +fi + +echo ${BF_HOSTNAME} > /etc/hostname + +hostname ${BF_HOSTNAME} + +/bin/rm -rf ${WDIR} +mkdir -p ${WDIR} +cd ${WDIR} +/bin/rm -rf initramfs + +initramfs=$WDIR/dump-initramfs-v0 + +dracut --no-hostonly -m "bash network base shutdown" --add-drivers "${bootctl_module} kmod sdhci-of-dwcmshc mlxbf_tmfifo dw_mmc-bluefield mmc_block virtio_console mlx5_core mlx5_ib ib_umad nvme gpio-mlxbf2 gpio-mlxbf3 mlxbf-gige" --filesystems "xfs vfat" --gzip $initramfs ${kernel} + +rm -f /etc/udev/rules.d/91-tmfifo_net.rules /etc/udev/rules.d/92-oob_net.rules + +cat > /etc/default/grub << EOF +GRUB_TIMEOUT=0 +GRUB_RECORDFAIL_TIMEOUT=2 +GRUB_DEFAULT=saved +GRUB_DISABLE_SUBMENU=true +GRUB_TERMINAL=console +GRUB_SERIAL_COMMAND="serial --unit=1" +GRUB_CMDLINE_LINUX="crashkernel=auto console=hvc0 console=ttyAMA0 earlycon=pl011,0x01000000 modprobe.blacklist=mlx5_core,mlx5_ib net.ifnames=0 biosdevname=0" +GRUB_DISABLE_RECOVERY="true" +GRUB_TIMEOUT_STYLE=countdown +GRUB_DISABLE_OS_PROBER=false +EOF + +case "$(file --brief --mime-type "$initramfs")" in + "application/x-lzma") + cat_initrd="lzcat" ;; + "application/x-lz4") + cat_initrd="lz4cat" ;; + *) + cat_initrd="zcat" ;; +esac + +echo "Rebuilding $initramfs" + +mkdir -p ${WDIR}/initramfs +cd initramfs +$cat_initrd "$initramfs" | cpio -i + +copy_bin() +{ + if [ -e $1 ]; then + bin=$1 + else + bin=`which $1 2> /dev/null` + fi + if [ -z "$bin" ]; then + echo "ERROR: Cannot find $1" + return 1 + fi + mkdir -p .`dirname $bin` + if [ -e .${bin} ]; then + /bin/rm -f .${bin} + fi + cp -a $bin .${bin} + + # Copy dependencies + for lib in `ldd $bin 2> /dev/null | grep '=>' | awk '{print $3}' | grep -v '(0x0'` + do + if [ -e .$lib ]; then + continue + fi + mkdir -p .`dirname $lib` + cp -a $lib .$lib + if [ -h $lib ]; then + tlib=`readlink -f $lib` + if [ ! -e .$tlib ]; then + mkdir -p .`dirname $tlib` + cp $tlib .$tlib + fi + fi + done +} + +copy_rpm() +{ + package=$1 + + if ! (rpm -q $package); then + yum install -y $package || return + fi + + case $package in + chkconfig) + for f in `rpm -ql $package | grep -v share` + do + if [ -d $f ]; then + mkdir -p .${f} + else + copy_bin $f + fi + done + ;; + *mft*) + for f in `rpm -ql $package | grep -v man` + do + if [ -d $f ]; then + continue + fi + copy_bin $f + done + ;; + *) + for f in `rpm -ql $package | grep -vE 'share|build-id'` + do + if [ -d $f ]; then + continue + fi + copy_bin $f + done + ;; + esac +} + +# Copy the content of RPM packages +for package in sdhci-of-dwcmshc mlxbf-bfscripts util-linux dosfstools e2fsprogs pciutils usbutils chkconfig mft mft-oem kernel-mft tar xfsprogs coreutils-single parted which curl openssl gawk passwd net-tools wget $ADDON_RPMS +do + copy_rpm $package || true +done + +# Copy tools +for tool in xz efibootmgr bash getopt hexdump mstflint lspci perl \ + lsblk shutdown systemctl strings \ + mlxbf-bootctl id grep $ADDON_TOOLS +do + if [ -d $tool ]; then + continue + fi + copy_bin $tool +done + +depmod -a -b ./ $kernel +ldconfig -r ./ +mkdir -p usr/share/hwdata/ bin/ +cp /usr/share/hwdata/pci.ids ./usr/share/hwdata/ +mkdir -p ./lib/firmware/mellanox/boot/ +cp /lib/firmware/mellanox/boot/default.bfb ./lib/firmware/mellanox/boot/default.bfb +cp -a /lib/firmware/mellanox/boot/capsule ./lib/firmware/mellanox/boot/ +cp $CAPSULE ./lib/firmware/mellanox/boot/capsule/ +mkdir -p mnt + +mkdir -p anolis +# Tar the image +XZ_OPT="--threads=0 -9 --verbose" tar -cJp --exclude='./workspace' --exclude='./signing-local' --exclude='./signing-shared' --exclude='./var/run/docker.sock' \ + --exclude='./root/workspace' --exclude='./tmp/*' --exclude='./run/*' --exclude='./sys/*' --exclude='./proc/*' --exclude='./dev/*' --exclude='./.dockerenv' \ + --exclude='./root/rpmbuild' --exclude='./root/.rpmmacros' --exclude='./root/.bash_history' --exclude='./var/tmp/*rpm' --exclude='./bin/qemu-aarch64-static' \ + --exclude='./usr/bin/qemu-aarch64-static' --exclude='./usr/bin/qemu-arm-static' --exclude='./var/lib/yum/*' --exclude='./var/cache/yum/*' -f "./anolis/image.tar.xz" -C / . +install -m 0755 ${SDIR}/install.sh ./anolis/install.sh + +mkdir scripts +cat > scripts/install-anolis.sh << EOF +#!/bin/sh + +echo +echo "=================================" | tee /dev/kmsg +echo "Installing Anolis Linux. Please wait..." | tee /dev/kmsg +echo "=================================" | tee /dev/kmsg + +modprobe -a sdhci-of-dwcmshc mlxbf_tmfifo dw_mmc-bluefield mmc_block virtio_console nvme ${bootctl_module} 2>&1 | tee /dev/kmsg +modprobe -a mlx5_ib ib_uverbs 2>&1 | tee /dev/kmsg + +sleep 5 + +/bin/sh /anolis/install.sh +if [ \$? -eq 0 ]; then + echo "===================================" | tee /dev/kmsg + echo "Installation finished. Rebooting..." | tee /dev/kmsg + echo "===================================" | tee /dev/kmsg + echo + reboot -f +else + echo "========================" | tee /dev/kmsg + echo "Failed to install Anolis Linux" | tee /dev/kmsg + echo "========================" | tee /dev/kmsg +fi +EOF + +sed -i -e "s@source_conf /etc/conf.d@&\nbash -x /scripts/install-anolis.sh@" init + +# Make initramfs +find . | cpio -H newc -o | gzip -9 > $initramfs + +# Make bfb +cd .. +if [ ! -e $CAPSULE ]; then + echo "ERROR: Capsule file $CAPSULE does not exist" + exit 1 +fi + +boot_args=$(mktemp) +boot_args2=$(mktemp) +boot_path=$(mktemp) +boot_desc=$(mktemp) +printf "console=ttyAMA1 console=hvc0 console=ttyAMA0 earlycon=pl011,0x01000000 earlycon=pl011,0x01800000 initrd=initramfs" > \ + "$boot_args" +printf "console=hvc0 console=ttyAMA0 earlycon=pl011,0x13010000 initrd=initramfs" > \ + "$boot_args2" + +printf "VenHw(F019E406-8C9C-11E5-8797-001ACA00BFC4)/Image" > "$boot_path" +printf "Linux from rshim" > "$boot_desc" +vmlinuz=$WDIR/vmlinuz +if [ -e $SDIR/vmlinuz ]; then + zcat $SDIR/vmlinuz > "$vmlinuz" +elif [ -e /boot/vmlinuz-$kernel ]; then + zcat /boot/vmlinuz-$kernel > "$vmlinuz" +else + zcat /lib/modules/$kernel/vmlinuz > "$vmlinuz" +fi + +mkdir -p /workspace + +$mkbfb \ + --image "$vmlinuz" --initramfs "$initramfs" \ + --capsule "$CAPSULE" \ + --boot-args-v0 "$boot_args" \ + --boot-args-v2 "$boot_args2" \ + --boot-path "$boot_path" \ + --boot-desc "$boot_desc" \ + ${BFB} /workspace/${BFB_FILENAME} +cd .. + +echo "BFB is ready: `readlink -f /workspace/${BFB_FILENAME}`" diff --git a/anolis/8.6/install.sh b/anolis/8.6/install.sh new file mode 100755 index 0000000..cb2f4cb --- /dev/null +++ b/anolis/8.6/install.sh @@ -0,0 +1,500 @@ +#!/bin/bash + +############################################################################### +# +# Copyright 2020 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +############################################################################### + +PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/mellanox/scripts" +NIC_FW_UPDATE_DONE=0 + +fspath=$(readlink -f `dirname $0`) + +rshimlog=`which bfrshlog 2> /dev/null` +log() +{ + msg="[$(date +%H:%M:%S)] $*" + echo "$msg" > /dev/ttyAMA0 + echo "$msg" > /dev/hvc0 + if [ -n "$rshimlog" ]; then + $rshimlog "$*" + fi +} + +fw_update() +{ + FW_UPDATER=/opt/mellanox/mlnx-fw-updater/mlnx_fw_updater.pl + FW_DIR=/opt/mellanox/mlnx-fw-updater/firmware/ + + if [[ -x /mnt/${FW_UPDATER} && -d /mnt/${FW_DIR} ]]; then + log "INFO: Updating NIC firmware..." + chroot /mnt ${FW_UPDATER} \ + --force-fw-update \ + --fw-dir ${FW_DIR} + if [ $? -eq 0 ]; then + log "INFO: NIC firmware update done" + else + log "INFO: NIC firmware update failed" + fi + else + log "WARNING: NIC Firmware files were not found" + fi +} + +fw_reset() +{ + mst start > /dev/null 2>&1 || true + chroot /mnt /sbin/mlnx_bf_configure > /dev/null 2>&1 + + MLXFWRESET_TIMEOUT=${MLXFWRESET_TIMEOUT:-180} + SECONDS=0 + while ! (chroot /mnt mlxfwreset -d /dev/mst/mt*_pciconf0 q 2>&1 | grep -w "Driver is the owner" | grep -qw "\-Supported") + do + if [ $SECONDS -gt $MLXFWRESET_TIMEOUT ]; then + log "INFO: NIC Firmware reset is not supported. Host power cycle is required" + return + fi + sleep 1 + done + + log "INFO: Running NIC Firmware reset" + if [ "X$mode" == "Xmanufacturing" ]; then + log "INFO: Rebooting..." + fi + # Wait for these messages to be pulled by the rshim service + # as mlxfwreset will restart the DPU + sleep 3 + + msg=`chroot /mnt mlxfwreset -d /dev/mst/mt*_pciconf0 -y -l 3 --sync 1 r 2>&1` + if [ $? -ne 0 ]; then + log "INFO: NIC Firmware reset failed" + log "INFO: $msg" + else + log "INFO: NIC Firmware reset done" + fi +} + +bind_partitions() +{ + mount --bind /proc /mnt/proc + mount --bind /dev /mnt/dev + mount --bind /sys /mnt/sys +} + +unmount_partitions() +{ + umount /mnt/sys/fs/fuse/connections > /dev/null 2>&1 || true + umount /mnt/sys > /dev/null 2>&1 + umount /mnt/dev > /dev/null 2>&1 + umount /mnt/proc > /dev/null 2>&1 + umount /mnt/boot/efi > /dev/null 2>&1 + umount /mnt > /dev/null 2>&1 +} + +# +# Set the Hardware Clock from the System Clock +# + +hwclock -w + +# +# Check auto configuration passed from boot-fifo +# +boot_fifo_path="/sys/bus/platform/devices/MLNXBF04:00/bootfifo" +if [ -e "${boot_fifo_path}" ]; then + cfg_file=$(mktemp) + # Get 16KB assuming it's big enough to hold the config file. + dd if=${boot_fifo_path} of=${cfg_file} bs=4096 count=4 > /dev/null 2>&1 + + # + # Check the .xz signature {0xFD, '7', 'z', 'X', 'Z', 0x00} and extract the + # config file from it. Then start decompression in the background. + # + offset=$(strings -a -t d ${cfg_file} | grep -m 1 "7zXZ" | awk '{print $1}') + if [ -s "${cfg_file}" -a ."${offset}" != ."1" ]; then + log "INFO: Found bf.cfg" + cat ${cfg_file} | tr -d '\0' > /etc/bf.cfg + fi + rm -f $cfg_file +fi + +# +# Check PXE installation +# +if [ ! -e /tmp/bfpxe.done ]; then touch /tmp/bfpxe.done; bfpxe; fi + +if [ -e /etc/bf.cfg ]; then + if ( bash -n /etc/bf.cfg ); then + . /etc/bf.cfg + else + log "INFO: Invalid bf.cfg" + fi +fi + +distro="Anolis" + +function_exists() +{ + declare -f -F "$1" > /dev/null + return $? +} + +DHCP_CLASS_ID=${PXE_DHCP_CLASS_ID:-""} +DHCP_CLASS_ID_OOB=${DHCP_CLASS_ID_OOB:-"NVIDIA/BF/OOB"} +DHCP_CLASS_ID_DP=${DHCP_CLASS_ID_DP:-"NVIDIA/BF/DP"} +FACTORY_DEFAULT_DHCP_BEHAVIOR=${FACTORY_DEFAULT_DHCP_BEHAVIOR:-"true"} + +if [ "${FACTORY_DEFAULT_DHCP_BEHAVIOR}" == "true" ]; then + # Set factory defaults + DHCP_CLASS_ID="NVIDIA/BF/PXE" + DHCP_CLASS_ID_OOB="NVIDIA/BF/OOB" + DHCP_CLASS_ID_DP="NVIDIA/BF/DP" +fi + +log "INFO: $distro installation started" + +# Create partitions. +default_device=/dev/mmcblk0 +if [ -b /dev/nvme0n1 ]; then + default_device=/dev/nvme0n1 +fi +device=${device:-"$default_device"} + +if [ "$SHRED_DRIVES" == "yes" ]; then + if [ -b /dev/nvme0n1 ]; then + log "Shredding /dev/nvme0n1" + shred -n 1 -v /dev/nvme0n1 & + fi + if [ -b /dev/mmcblk0 ]; then + log "Shredding /dev/mmcblk0" + shred -n 1 -v /dev/mmcblk0 & + fi + wait +fi + +dd if=/dev/zero of=$device bs=512 count=1 + +parted --script $device -- \ + mklabel gpt \ + mkpart primary 1MiB 201MiB set 1 esp on \ + mkpart primary 201MiB 1225MiB \ + mkpart primary 1225MiB 100% + +sync + +partprobe "$device" > /dev/null 2>&1 + +sleep 1 +blockdev --rereadpt "$device" > /dev/null 2>&1 + +if function_exists bfb_pre_install; then + log "INFO: Running bfb_pre_install from bf.cfg" + bfb_pre_install +fi + +# Generate some entropy +mke2fs ${device}p2 >> /dev/null + +# Copy the kernel image. +mkdosfs ${device}p1 -n system-boot +mkfs.xfs -f ${device}p2 -L local-boot +mkfs.xfs -f ${device}p3 -L writable + +export EXTRACT_UNSAFE_SYMLINKS=1 + +fsck.vfat -a ${device}p1 + +root=${device/\/dev\/}p3 +mount ${device}p3 /mnt +mkdir -p /mnt/boot +mount ${device}p2 /mnt/boot +mkdir -p /mnt/boot/efi +mount ${device}p1 /mnt/boot/efi + +echo "Extracting /..." +tar Jxf $fspath/image.tar.xz --warning=no-timestamp -C /mnt +sync + +cat > /mnt/etc/fstab << EOF +# +# /etc/fstab +# +# +${device}p3 / xfs defaults 0 1 +${device}p2 /boot xfs defaults 0 2 +${device}p1 /boot/efi vfat umask=0077,shortname=winnt 0 2 +EOF + +memtotal=$(awk '/MemTotal/ {print $2}' /proc/meminfo) +if [ $memtotal -gt 16000000 ]; then + sed -i -r -e "s/(net.netfilter.nf_conntrack_max).*/\1 = 1000000/" /mnt/usr/lib/sysctl.d/90-bluefield.conf +fi + +cat > /mnt/etc/udev/rules.d/50-dev-root.rules << EOF +# If the system was booted without an initramfs, grubby +# will look for the symbolic link "/dev/root" to figure +# out the root file system block device. +SUBSYSTEM=="block", KERNEL=="$root", SYMLINK+="root" +EOF + +# Update default.bfb +bfb_location=/lib/firmware/mellanox/default.bfb + +if [ -f "$bfb_location" ]; then + /bin/rm -f /mnt/lib/firmware/mellanox/boot/default.bfb + cp $bfb_location /mnt/lib/firmware/mellanox/boot/default.bfb +fi + +# Disable SELINUX +sed -i -e "s/^SELINUX=.*/SELINUX=disabled/" /mnt/etc/selinux/config + +chmod 600 /mnt/etc/ssh/* + +# Disable Firewall services +/bin/rm -f /mnt/etc/systemd/system/multi-user.target.wants/firewalld.service +/bin/rm -f /mnt/etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service + +bind_partitions + +/bin/rm -f /mnt/boot/vmlinux-*.bz2 + +# Then, set boot arguments: Read current 'console' and 'earlycon' +# parameters, and append the root filesystem parameters. +bootarg="$(cat /proc/cmdline | sed 's/initrd=initramfs//;s/console=.*//')" +sed -i -e "s@GRUB_CMDLINE_LINUX=.*@GRUB_CMDLINE_LINUX=\"rw crashkernel=auto $bootarg console=hvc0 console=ttyAMA0 earlycon=pl011,0x01000000 modprobe.blacklist=mlx5_core,mlx5_ib net.ifnames=0 biosdevname=0 iommu.passthrough=1\"@" /mnt/etc/default/grub +if (hexdump -C /sys/firmware/acpi/tables/SSDT* | grep -q MLNXBF33); then + # BlueField-3 + sed -i -e "s/0x01000000/0x13010000/g" /mnt/etc/default/grub +fi + +chroot /mnt grub2-mkconfig -o /boot/efi/EFI/anolis/grub.cfg + +kdir=$(/bin/ls -1d /mnt/lib/modules/4.18* /mnt/lib/modules/4.19* /mnt/lib/modules/4.20* /mnt/lib/modules/5.* 2> /dev/null) +kver="" +if [ -n "$kdir" ]; then + kver=${kdir##*/} + DRACUT_CMD=`chroot /mnt /bin/ls -1 /sbin/dracut /usr/bin/dracut 2> /dev/null | head -n 1 | tr -d '\n'` + chroot /mnt grub2-set-default 0 + chroot /mnt $DRACUT_CMD --kver ${kver} --force --add-drivers "sdhci-of-dwcmshc dw_mmc-bluefield dw_mmc dw_mmc-pltfm mmc_block mlxbf_tmfifo virtio_console nvme" /boot/initramfs-${kver}.img +else + kver=$(/bin/ls -1 /mnt/lib/modules/ | head -1) +fi + +echo anolis | chroot /mnt passwd root --stdin + +if [ `wc -l /mnt/etc/hostname | cut -d ' ' -f 1` -eq 0 ]; then + echo "localhost" > /mnt/etc/hostname +fi + +cat > /mnt/etc/resolv.conf << EOF +nameserver 192.168.100.1 +EOF + +chroot /mnt /bin/systemctl unmask dev-hugepages.mount sys-fs-fuse-connections.mount console-getty.service dbus-org.freedesktop.login1.service systemd-logind.service systemd-remount-fs.service getty.target +chroot /mnt /bin/systemctl enable serial-getty@ttyAMA0.service serial-getty@ttyAMA1.service serial-getty@hvc0.service +chroot /mnt /bin/systemctl disable getty@tty0.service rshim-getty.service +chroot /mnt /bin/systemctl add-wants default.target getty.target + +if [ -x /usr/bin/uuidgen ]; then + UUIDGEN=/usr/bin/uuidgen +else + UUIDGEN=/mnt/usr/bin/uuidgen +fi + +p0m0_uuid=`$UUIDGEN` +p1m0_uuid=`$UUIDGEN` +p0m0_mac=`echo ${p0m0_uuid} | sed -e 's/-//;s/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'` +p1m0_mac=`echo ${p1m0_uuid} | sed -e 's/-//;s/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'` + +pciids=`lspci -nD 2> /dev/null | grep 15b3:a2d[26c] | awk '{print $1}'` + +mkdir -p /mnt/etc/mellanox +echo > /mnt/etc/mellanox/mlnx-sf.conf + +i=0 +for pciid in $pciids +do + uuid_iname=p${i}m0_uuid + mac_iname=p${i}m0_mac +cat >> /mnt/etc/mellanox/mlnx-sf.conf << EOF +/sbin/mlnx-sf --action create --device $pciid --sfnum 0 --hwaddr ${!mac_iname} +EOF + let i=i+1 +done + +# Update HW-dependant files +if (lspci -n -d 15b3: | grep -wq 'a2d2'); then + # BlueField-1 + ln -snf snap_rpc_init_bf1.conf /mnt/etc/mlnx_snap/snap_rpc_init.conf + # OOB interface does not exist on BlueField-1 + /bin/rm -f /mnt/etc/sysconfig/network-scripts/ifcfg-oob_net0 +elif (lspci -n -d 15b3: | grep -wq 'a2d6'); then + # BlueField-2 + ln -snf snap_rpc_init_bf2.conf /mnt/etc/mlnx_snap/snap_rpc_init.conf +elif (lspci -n -d 15b3: | grep -wq 'a2dc'); then + # BlueField-3 + chroot /mnt rpm -e mlnx-snap || true +fi + + mkdir -p /mnt/etc/dhcp + cat >> /mnt/etc/dhcp/dhclient.conf << EOF +send vendor-class-identifier "$DHCP_CLASS_ID_DP"; +interface "oob_net0" { + send vendor-class-identifier "$DHCP_CLASS_ID_OOB"; +} +EOF + +# Customisations per PSID +FLINT="" +if [ -x /usr/bin/mstflint ]; then + FLINT=/usr/bin/mstflint +elif [ -x /usr/bin/flint ]; then + FLINT=/usr/bin/flint +elif [ -x /mnt/usr/bin/mstflint ]; then + FLINT=/mnt/usr/bin/mstflint +fi + +pciid=`echo $pciids | awk '{print $1}' | head -1` +if [ -e /mnt/usr/sbin/mlnx_snap_check_emulation.sh ]; then + sed -r -i -e "s@(NVME_SF_ECPF_DEV=).*@\1${pciid}@" /mnt/usr/sbin/mlnx_snap_check_emulation.sh +fi +if [ -n "$FLINT" ]; then + PSID=`$FLINT -d $pciid q | grep PSID | awk '{print $NF}'` + + case "${PSID}" in + MT_0000000634) + sed -r -i -e 's@(EXTRA_ARGS=).*@\1"--mem-size 1200"@' /mnt/etc/default/mlnx_snap + ;; + esac +fi + +if [ "$WITH_NIC_FW_UPDATE" == "yes" ]; then + if [ $NIC_FW_UPDATE_DONE -eq 0 ]; then + fw_update + NIC_FW_UPDATE_DONE=1 + fi +fi + +# Clean up logs +echo > /mnt/var/log/messages +echo > /mnt/var/log/maillog +echo > /mnt/var/log/secure +echo > /mnt/var/log/firewalld +echo > /mnt/var/log/audit/audit.log +/bin/rm -f /mnt/var/log/yum.log +/bin/rm -rf /mnt/tmp/* + +if function_exists bfb_modify_os; then + log "INFO: Running bfb_modify_os from bf.cfg" + bfb_modify_os +fi + +sync + +chroot /mnt umount /boot/efi +chroot /mnt umount /boot +umount /mnt/sys +umount /mnt/dev +umount /mnt/proc +umount /mnt + +sync + +UPDATE_BOOT=${UPDATE_BOOT:-1} +if [ $UPDATE_BOOT -eq 1 ]; then + bfrec --bootctl 2> /dev/null || true + if [ -e /lib/firmware/mellanox/boot/capsule/boot_update2.cap ]; then + bfrec --capsule /lib/firmware/mellanox/boot/capsule/boot_update2.cap + fi + + if [ -e /lib/firmware/mellanox/boot/capsule/efi_sbkeysync.cap ]; then + bfrec --capsule /lib/firmware/mellanox/boot/capsule/efi_sbkeysync.cap + fi +fi + +# Clean up actual boot entries. +bfbootmgr --cleanall > /dev/null 2>&1 + +if [ ! -d /sys/firmware/efi/efivars ]; then + mount -t efivarfs none /sys/firmware/efi/efivars +fi + +/bin/rm -f /sys/firmware/efi/efivars/Boot* > /dev/null 2>&1 +/bin/rm -f /sys/firmware/efi/efivars/dump-* > /dev/null 2>&1 +efibootmgr -c -d "$device" -p 1 -l "\EFI\anolis\shimaa64.efi" -L $distro +umount /sys/firmware/efi/efivars + +BFCFG=`which bfcfg 2> /dev/null` +if [ -n "$BFCFG" ]; then + # Create PXE boot entries + if [ -e /etc/bf.cfg ]; then + mv /etc/bf.cfg /etc/bf.cfg.orig + fi + + cat > /etc/bf.cfg << EOF +BOOT0=DISK +BOOT1=NET-NIC_P0-IPV4 +BOOT2=NET-NIC_P0-IPV6 +BOOT3=NET-NIC_P1-IPV4 +BOOT4=NET-NIC_P1-IPV6 +BOOT5=NET-OOB-IPV4 +BOOT6=NET-OOB-IPV6 +PXE_DHCP_CLASS_ID=$DHCP_CLASS_ID +EOF + + $BFCFG + + # Restore the original bf.cfg + /bin/rm -f /etc/bf.cfg + if [ -e /etc/bf.cfg.orig ]; then + grep -v PXE_DHCP_CLASS_ID= /etc/bf.cfg.orig > /etc/bf.cfg + fi +fi + +if [ -n "$BFCFG" ]; then + $BFCFG +fi + +echo +echo "ROOT PASSWORD is \"anolis\"" +echo + +if function_exists bfb_post_install; then + log "INFO: Running bfb_post_install from bf.cfg" + bfb_post_install +fi + +log "INFO: Installation finished" + +if [ "$WITH_NIC_FW_UPDATE" == "yes" ]; then + if [ $NIC_FW_UPDATE_DONE -eq 1 ]; then + # Reset NIC FW + mount ${device}p3 /mnt + bind_partitions + fw_reset + unmount_partitions + fi +fi + +sleep 3 +log "INFO: Rebooting..." +# Wait for these messages to be pulled by the rshim service +sleep 3