default-package-config.sh

#!/usr/bin/env bash
#
# Copyright 2018, 2020 Delphix
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# shellcheck disable=SC2034

#
# This file defines a default config for a package and is sourced by
# lib/common.sh->load_package_config() before sourcing the config.sh
# for the specified package. Any hooks and variables defined here can
# be overriden.
#

function fetch() {
	logmust fetch_repo_from_git
}

function merge_with_upstream() {
	logmust merge_with_upstream_default
}

#
# The functions below are specific for the Linux kernel packages
# and contain the majority of their common code.
#
function kernel_prepare() {
	logmust install_pkgs \
		equivs \
		devscripts \
		kernel-wedge
}

#
# The configuration disabled below is specifically for uses
# of ${debian_rules_args}. Quoting the specific variable
# results in incorrect behavior and thus we disable that
# check.
#
# shellcheck disable=SC2086
function kernel_build() {
	local platform="$1"
	#
	# Note: Extra arguments can overwrite default arguments.
	#       For example in this function we default skipdbg
	#       to false, but if we pass "skipdbg=true" as an
	#       extra argument we will be overwriting this value
	#       to true. This is because when a variable's value
	#       is declared multiple times when invoking the
	#       debian/rules command, the rightmost declaration
	#       is the one that is actually used.
	#
	shift
	local debian_rules_extra_args=("$@")

	logmust cd "$WORKDIR/repo"

	#
	# We generate the default control file from Canonical
	# so we can capture the ABI number (abinum) from
	# Canonical's kernel - (see comment that follows for
	# the reason and the relevant code for the logic).
	#
	logmust debian/rules debian/control

	#
	# We overwrite the default abinum build variable with our
	# version strings and at the same time retain the original
	# abinum from Canonical by appending it at the end.
	#
	# We chose to mutate the abinum field as it is the least
	# invasive for Ubuntu's build logic (e.g. most of the other
	# variables actually interact with logic in the build). At
	# the same time the abinum variable is part of the fields
	# that we care about (e.g. package name, linux image file
	# name, etc..).
	#
	# We still retain the original abinum by appending it at
	# the end of the new one to maintain the mapping between
	# Canonical's releases and our releases.
	#
	local canonical_abinum delphix_abinum kernel_release kernel_version
	canonical_abinum=$(fakeroot debian/rules printenv | grep -E '^abinum ' | cut -d= -f2 | tr -d '[:space:]')
	delphix_abinum="${canonical_abinum}-$(date -u +"dx%Y%m%d%H")-$(git rev-parse --short HEAD)"
	kernel_release=$(fakeroot debian/rules printenv | grep -E '^release ' | cut -d= -f2 | tr -d '[:space:]')

	#
	# We record the kernel version into a file. This field is consumed
	# by other kernel packages, such as zfs, during their build.
	#
	kernel_version="${kernel_release}-${delphix_abinum}-${platform}"
	echo "$kernel_version" >"$WORKDIR/artifacts/KERNEL_VERSION"

	#
	# skipdbg=false
	#   We need debug info for our debugging tools to work.
	#   Don't skip them.
	# uefi_signed=false
	#   This variable defaults to true but since we don't have
	#   any intention and logic to provide signatures for now
	#   we set it to false to avoid any misconfigurations down
	#   the line.
	# disable_d_i=true
	#   This prevents udeb packages from being built as they are
	#   not consumed by the Delphix Appliance.
	# do_dkms_*=false
	#   This disables the build of various out-of-tree kernel modules
	#   that we do not use in our product or that we provide separately.
	#
	local debian_rules_args=(
		"skipdbg=false"
		"uefi_signed=false"
		"disable_d_i=true"
		"do_zfs=false"
		"do_dkms_nvidia=false"
		"do_dkms_nvidia_server=false"
		"do_dkms_vbox=false"
		"do_dkms_wireguard=false"
		"dkms_exclude=v4l2loopback"
		"flavours=$platform"
		"abinum=${delphix_abinum}"
	)
	debian_rules_args+=("${debian_rules_extra_args[@]}")

	#
	# Clean up everything generated so far and recreate the
	# final control file with the arguments that we want.
	#
	logmust fakeroot debian/rules clean "${debian_rules_args[@]}"

	#
	# Print the environment configuration solely for
	# debugging purposes.
	#
	logmust fakeroot debian/rules printenv "${debian_rules_args[@]}"

	#
	# The default value of the tool argument for mk-build-deps
	# is the following:
	# "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends"
	#
	# We append --yes to it to disable interactivity by apt-get
	# and allow for automation.
	#
	local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes"
	logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}"

	#
	# Here we update the configs used to control the kernel's build
	# system. This is useful as it allows us to override various
	# kernel config options via an annotations file, which we use to
	# disable various kernel modules that we don't need or want.
	# The kernel provides a mechanism to manipulate the annotations via
	# a python script at `debian/scripts/misc/annotations`. The script
	# is first used to export the Delphix's annotations file to an old-style
	# config, which in turn is imported back into the kernel's base
	# annotations. This allows us to maintain a partial set of annotations
	# that are only relevant to our product. At the time of writing this,
	# there was no direct way to import a partial annotations file into the base
	# annotations file without first converting it to an old-style config.
	#
	. debian/debian.env
	local architecture
	architecture=$(dpkg-architecture -q DEB_HOST_ARCH 2>/dev/null)
	logmust debian/scripts/misc/annotations -f "${WORKDIR}/../../../resources/delphix_kernel_annotations" --arch "${architecture}" --flavour "${platform}" --export >delphix.config
	logmust debian/scripts/misc/annotations -f ${DEBIAN}/config/annotations --arch "${architecture}" --flavour "${platform}" --update delphix.config

	logmust fakeroot debian/rules updateconfigs "${debian_rules_args[@]}" do_skip_checks=true

	logmust fakeroot debian/rules "binary" "${debian_rules_args[@]}"

	logmust cd "$WORKDIR"
	logmust mv ./*deb "artifacts/"

	#
	# Make sure that we recorded the kernel version properly by checking
	# one of the .debs produced
	#
	logmust test -f "artifacts/linux-image-${kernel_version}_"*.deb
}

#
# Syncing our kernel with the right upstream Canonical repo is not as
# straighforward as the other packages in linux-pkg.
#
# The Ubuntu developers maintain the timeline of the mainline kernel
# (kernel.org) in their git history. When it is time to use a new
# mainline version, they fork a new branch and then cherry-pick all their
# Ubuntu-specific generic patches on top of that and create their base
# tag. Then on top of that they cherry-pick their platform-specific
# patches (e.g. azure, aws, etc..) and create separate tags for each
# platform. This whole process is repeated for every bump in the kernel
# version (both mainline and ubuntu-specific).
#
# We want to track and sync our changes every time Canonical bumps
# their kernel version for the kernels that are used by the Delphix
# Engine that we release in order to stay up to date and lower our
# maintainance burden. As a result we do the following:
#
# * We have one kernel repo per platform.
# * Each of this repos is an Ubuntu kernel repo with our specific
#   patches on top.
# * The vanilla Ubuntu kernel and our patches are divided by a single
#   placeholder commit with the description "@@DELPHIX_PATCHSET_START@@".
# * Whenever the Ubuntu kernel version is bumped, we detect that
#   change and use the new Ubuntu version as our base and cherry-pick
#   the placeholder commit followed by our patches on top of it.
#
function kernel_update_upstream() {
	local platform="$1"

	check_env UPSTREAM_GIT_URL
	logmust cd "$WORKDIR/repo"

	#
	# checkout our local branch that tracks upstream.
	#
	logmust git checkout -q upstream-HEAD

	#
	# declare third-party upstream repository.
	#
	logmust git remote add upstream "$UPSTREAM_GIT_URL"

	#
	# We get the kernel version and the ABI number from
	# $_RET that is set from `get_kernel_from_platform()`.
	# Example:
	#
	#   $_RET -> 5.3.0-53-generic
	#
	#   `cut -d '-' -f 1` of that -> 5.3.0
	#   `cut -d '-' -f 3` of that -> 53
	#
	# We need the kernel version and ABI number to figure
	# out the latest upstream tag to sync with.
	#
	local kernel_version abinum
	logmust get_kernel_version_for_platform_from_apt "${platform}"
	kernel_version=$(echo "$_RET" | cut -d '-' -f 1)
	abinum=$(echo "$_RET" | cut -d '-' -f 2)

	#
	# For each supported platform we will try to find the
	# latest upstream tag to sync based on the kernel
	# version and the ABI num that we got above.
	#
	# Note that "generic" (used mainly ESX) is a special
	# case on bionic where we are using the HWE kernel image.
	#
	local tag_prefix_flavour
	case "${platform}" in
	generic)
		if [[ "$UBUNTU_DISTRIBUTION" == focal ]]; then
			tag_prefix_flavour="Ubuntu-hwe"
		else
			tag_prefix_flavour="Ubuntu"
		fi
		;;
	aws | azure | gcp | oracle)
		tag_prefix_flavour="Ubuntu-${platform}"
		;;
	*)
		die "assertion: unexpected platform: ${platform}"
		;;
	esac

	local tag_prefix kvers_major kvers_minor short_kvers
	kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
	kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
	short_kvers="${kvers_major}.${kvers_minor}"
	tag_prefix="${tag_prefix_flavour}-${short_kvers}-${kernel_version}-${abinum}"
	echo "note: upstream tag prefix used: ${tag_prefix}"

	#
	# Query for upstream tag info based on the prefix that we've
	# assembled.
	#
	# = Why the `tail -n 1` part?
	#
	# Using `git ls-remote` and `grep` with the tag's prefix alone
	# may sometimes return two (and theoretically more?) results
	# due to Ubuntu's "point releases". Point releases are specific
	# to LTS releases and more info about them can be found in the
	# links below:
	# [1] https://wiki.ubuntu.com/LTS
	# [2] https://wiki.ubuntu.com/PointReleaseProcess
	#
	# Example:
	# ```
	# $ git ls-remote --tags upstream | grep Ubuntu-oracle-5.3-5.3.0-1015
	# df8fd7d8802d59   refs/tags/Ubuntu-oracle-5.3-5.3.0-1015.16_18.04.1
	# 0fe5cd29e90a5e   refs/tags/Ubuntu-oracle-5.3-5.3.0-1015.16_18.04.2
	# ```
	#
	# We most probably want the latest point release of a specific
	# kernel thus we add `tail -n 1` in the pipeline below.
	#
	local upstream_tag_info
	upstream_tag_info=$(git ls-remote --tags --ref upstream | grep "${tag_prefix}" | tail -n 1)

	if [[ -z "${upstream_tag_info}" ]]; then
		echo "tag with prefix ${tag_prefix} not found."
		tag_prefix="${tag_prefix_flavour}-${kernel_version}-${abinum}"
		#
		# Canonical has 2 ways of naming their tags:
		# - Ubuntu-gcp-5.4.0-1046.49
		# - Ubuntu-gcp-5.4-5.4.0-1046.49_18.04.1
		#
		# For a given kernel version and a given distribution, only one
		# naming scheme is being used. When two distributions (such
		# as 20.04 and 18.04) use the same major kernel version,
		# Canonical first applies the patches to one distribution and
		# then creates a base tag. It then backports those changes to
		# the older distribution, and then creates a longer tag name.
		#
		echo "trying tag prefix: ${tag_prefix}."
		upstream_tag_info=$(git ls-remote --tags --ref upstream | grep "${tag_prefix}" | tail -n 1)
	fi

	[[ -z "${upstream_tag_info}" ]] && die "could not find upstream tag for tag prefix: ${tag_prefix}"

	local upstream_tag
	upstream_tag=$(echo "${upstream_tag_info}" | awk -F / '{print $3}')
	[[ -z "${upstream_tag}" ]] && die "could not extract upstream tag name from the tag info"

	logmust git fetch upstream "+refs/tags/${upstream_tag}:refs/tags/${upstream_tag}"

	#
	# Note that we add '^{}' at the end to dereference the tag recursively
	# until it arrives to an actual commit. This is needed in case the
	# tag points to an anotated tag object which contains extra information
	# such as a PGP signature. That annoted tag will in turn reference the
	# actual commit, which will be returned when appending ^{}.
	# upstream-HEAD will be pointing to the commit directly rather than the
	# annoted tag object, so if we want to compare the two we need to query
	# for the dereferenced commit. See 'git help gitrevisions' for more info.
	#
	local upstream_tag_commit
	upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}^{}")" ||
		die "couldn't get commit of tag ${upstream_tag}"
	echo "note: upstream tag: ${upstream_tag}, commit ${upstream_tag_commit}"

	#
	# Check if the commit of the latest tag from upstream matches
	# what we have cached in our repository at upstreams/<branch>,
	# which we fetch to upstream-HEAD.
	#
	local local_upstream_commit
	local_upstream_commit=$(git rev-parse upstream-HEAD)
	[[ -z "${local_upstream_commit}" ]] && die "could not find upstream-HEAD's commit"
	check_env DEFAULT_GIT_BRANCH
	echo "note: upstreams/${DEFAULT_GIT_BRANCH} commit: ${local_upstream_commit}"

	if [[ "${upstream_tag_commit}" == "${local_upstream_commit}" ]]; then
		echo "NOTE: upstream for $PACKAGE is already up-to-date."
	else
		logmust git reset --hard "refs/tags/${upstream_tag}"
		echo "NOTE: upstream updated to refs/tags/${upstream_tag}"

		#
		# Store name of upstream tag so that we can push it to our
		# repository for reference purposes.
		#
		echo "refs/tags/${upstream_tag}" >"$WORKDIR/upstream-tag" ||
			die "failed to write to $WORKDIR/upstream-tag"

		logmust touch "$WORKDIR/upstream-updated"
	fi

	logmust cd "$WORKDIR"
}

#
# This merges local changes in repo-HEAD with upstream changes in upstream-HEAD.
# As opposed to the default merge function merge_with_upstream_default(), this
# uses git cherry-pick to rebase our changes on top of the upstream changes.
#
function kernel_merge_with_upstream() {
	local repo_ref="refs/heads/repo-HEAD"
	local upstream_ref="refs/heads/upstream-HEAD"

	logmust cd "$WORKDIR/repo"

	check_git_ref "$upstream_ref" "$repo_ref"

	#
	# Ensure that there is a commit marking the start of
	# the Delphix set of patches. Then get the hash of
	# the commit right before it.
	#
	local dlpx_patch_end dlpx_patch_start current_ubuntu_commit upstream_head_commit
	dlpx_patch_start=$(git log --pretty=oneline repo-HEAD | grep @@DELPHIX_PATCHSET_START@@ | awk '{ print $1 }')
	[[ -z "${dlpx_patch_start}" ]] && die "could not find DELPHIX_PATCHSET_START"
	[[ $(wc -l <<<"${dlpx_patch_start}") != 1 ]] && die "multiple DELPHIX_PATCHSET_START commits - ${dlpx_patch_start}"
	current_ubuntu_commit=$(git rev-parse "${dlpx_patch_start}"^)
	[[ -z "${current_ubuntu_commit}" ]] && die "could not find commit before DELPHIX_PATCHSET_START"
	dlpx_patch_end=$(git rev-parse repo-HEAD)
	[[ -z "${dlpx_patch_end}" ]] && die "could not find repo-HEAD's head commit"

	upstream_head_commit="$(git rev-parse "$upstream_ref")"

	if [[ "$current_ubuntu_commit" == "$upstream_head_commit" ]]; then
		echo "NOTE: $PACKAGE is already up-to-date with upstream."
		return 0
	fi

	#
	# We rebase all the Delphix commits on top of the new upstream-HEAD
	# by using git cherry-pick. Note that we also save the previous
	# tip of the active branch to repo-HEAD-saved as this reference will be
	# checked later by push-merge.sh.
	#

	logmust git branch repo-HEAD-saved repo-HEAD
	logmust git branch -D repo-HEAD
	logmust git checkout -q -b repo-HEAD upstream-HEAD

	# shellcheck disable=SC2086
	logmust git cherry-pick ${dlpx_patch_start}^..${dlpx_patch_end}

	logmust touch "$WORKDIR/repo-updated"
}

function post_build_checks() {

	# This function checks for SKIP_COPYRIGHTS_CHECK flag
	# in config.sh file of each package. If the flag is
	# present and is set to 'true', the check will be skipped.
	# The license information for the platform packages are
	# generated based on Ubuntu package convention and are
	# picked from copyright file in debian package. As a
	# part of the check we look for existance of the file in
	# each package.
	if [[ "$SKIP_COPYRIGHTS_CHECK" != true ]]; then
		cd "$WORKDIR/artifacts" || return

		set -o pipefail
		for deb in *.deb; do
			echo "Running: dpkg-deb -c $deb | grep '/usr/share/doc/' | grep copyright"
			dpkg-deb -c "$deb" | grep '/usr/share/doc/' | grep copyright || die "copyright file missing for package $deb"
		done
		set +o pipefail
	fi
}