Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/6.0/release' into 6.0/patch
Browse files Browse the repository at this point in the history
  • Loading branch information
rasantel committed May 13, 2021
2 parents 019a1e6 + 262dd13 commit 0c80c44
Show file tree
Hide file tree
Showing 12 changed files with 350 additions and 113 deletions.
60 changes: 43 additions & 17 deletions default-package-config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ function kernel_build() {
# debian/rules command, the rightmost declaration
# is the one that is actually used.
#
local debian_rules_extra_args="$2"
shift
local debian_rules_extra_args=("$@")

logmust cd "$WORKDIR/repo"

Expand Down Expand Up @@ -91,7 +92,7 @@ function kernel_build() {
#
local canonical_abinum delphix_abinum kernel_release kernel_version
canonical_abinum=$(fakeroot debian/rules printenv | grep -E '^abinum ' | cut -d= -f2 | tr -d '[:space:]')
delphix_abinum="dlpx-$(date -u +"%Y%m%dt%H%M%S")-$(git rev-parse --short HEAD)-${canonical_abinum}"
delphix_abinum="${canonical_abinum}-$(date -u +"dx%Y%m%d%H")-$(git rev-parse --short HEAD)"
kernel_release=$(fakeroot debian/rules printenv | grep -E '^release ' | cut -d= -f2 | tr -d '[:space:]')

#
Expand All @@ -113,20 +114,35 @@ function kernel_build() {
# disable_d_i=true
# This prevents udeb packages from being built as they are
# not consumed by the Delphix Appliance.
#
local debian_rules_args="skipdbg=false uefi_signed=false disable_d_i=true flavours=$platform abinum=${delphix_abinum} ${debian_rules_extra_args}"
# do_dkms_*=false
# This disables the build of various out-of-tree kernel modules
# that we do not use in our product or that we provide separately.
#
local debian_rules_args=(
"skipdbg=false"
"uefi_signed=false"
"disable_d_i=true"
"do_zfs=false"
"do_dkms_nvidia=false"
"do_dkms_nvidia_server=false"
"do_dkms_vbox=false"
"do_dkms_wireguard=false"
"flavours=$platform"
"abinum=${delphix_abinum}"
)
debian_rules_args+=("${debian_rules_extra_args[@]}")

#
# Clean up everything generated so far and recreate the
# final control file with the arguments that we want.
#
logmust fakeroot debian/rules clean ${debian_rules_args}
logmust fakeroot debian/rules clean "${debian_rules_args[@]}"

#
# Print the environment configuration solely for
# debugging purposes.
#
logmust fakeroot debian/rules printenv ${debian_rules_args}
logmust fakeroot debian/rules printenv "${debian_rules_args[@]}"

#
# The default value of the tool argument for mk-build-deps
Expand All @@ -139,7 +155,7 @@ function kernel_build() {
local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes"
logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}"

logmust fakeroot debian/rules "binary" ${debian_rules_args}
logmust fakeroot debian/rules "binary" "${debian_rules_args[@]}"

logmust cd "$WORKDIR"
logmust mv ./*deb "artifacts/"
Expand Down Expand Up @@ -220,24 +236,24 @@ function kernel_update_upstream() {
# Note that "generic" (used mainly ESX) is a special
# case as we are currently using the HWE kernel image.
#
local tag_prefix
local tag_prefix_flavour
if [[ "${platform}" == generic ]] &&
[[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then
tag_prefix="Ubuntu-hwe-${kernel_version}-${abinum}"
tag_prefix_flavour="Ubuntu-hwe"
elif [[ "${platform}" == aws ]] ||
[[ "${platform}" == azure ]] ||
[[ "${platform}" == gcp ]] ||
[[ "${platform}" == oracle ]]; then

local kvers_major kvers_minor short_kvers
kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
short_kvers="${kvers_major}.${kvers_minor}"

tag_prefix="Ubuntu-${platform}-${short_kvers}-${kernel_version}-${abinum}"
tag_prefix_flavour="Ubuntu-${platform}"
else
die "assertion: unexpected platform: ${platform}"
fi

local tag_prefix kvers_major kvers_minor short_kvers
kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
short_kvers="${kvers_major}.${kvers_minor}"
tag_prefix="${tag_prefix_flavour}-${short_kvers}-${kernel_version}-${abinum}"
echo "note: upstream tag prefix used: ${tag_prefix}"

#
Expand Down Expand Up @@ -274,8 +290,18 @@ function kernel_update_upstream() {

logmust git fetch upstream "+refs/tags/${upstream_tag}:refs/tags/${upstream_tag}"

#
# Note that we add '^{}' at the end to dereference the tag recursively
# until it arrives to an actual commit. This is needed in case the
# tag points to an anotated tag object which contains extra information
# such as a PGP signature. That annoted tag will in turn reference the
# actual commit, which will be returned when appending ^{}.
# upstream-HEAD will be pointing to the commit directly rather than the
# annoted tag object, so if we want to compare the two we need to query
# for the dereferenced commit. See 'git help gitrevisions' for more info.
#
local upstream_tag_commit
upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}")" ||
upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}^{}")" ||
die "couldn't get commit of tag ${upstream_tag}"
echo "note: upstream tag: ${upstream_tag}, commit ${upstream_tag_commit}"

Expand Down
27 changes: 23 additions & 4 deletions lib/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export UBUNTU_DISTRIBUTION="bionic"
# 2. "archive": dowloading from apt
# 3. "prebuilt": pre-built kernel stored in artifactory
#
export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="prebuilt"
export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="delphix"

# shellcheck disable=SC2086
function enable_colors() {
Expand Down Expand Up @@ -89,7 +89,7 @@ function die() {
}

function logmust() {
echo Running: "$@"
[[ "$LOGGING" == "false" ]] || echo Running: "$@" >&2
"$@" || die "failed command '$*'"
}

Expand Down Expand Up @@ -470,7 +470,7 @@ function install_pkgs() {
echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
"apt-get install -y $*"
sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
-y "$@" && return
-y --allow-downgrades "$@" && return
echo "apt-get install failed, retrying."
sleep 10
done
Expand Down Expand Up @@ -1116,7 +1116,26 @@ function fetch_kernel_from_artifactory() {
url="$url/linux-pkg/linux-prebuilt/${artifactory_deb}"

logmust cd "$WORKDIR/artifacts"
logmust wget -nv "$url"
fetch_file_from_artifactory "$url"
}

function fetch_file_from_artifactory() {
local url="$1"
local sha256_expected="$2"
local file sha256_actual

[[ -n "$url" ]] || die "url argument is missing."
file="$(basename "$url")"

logmust wget -nv "$url" -O "$file"

if [[ -n "$sha256_expected" ]]; then
sha256_actual="$(sha256sum "$file" | awk '{print $1}')"
if [[ "$sha256_expected" != "$sha256_actual" ]]; then
die "SHA256 mismatch. Expected: $sha256_expected," \
"Actual: $sha256_actual"
fi
fi
}

#
Expand Down
4 changes: 3 additions & 1 deletion package-lists/build/main.pkgs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ grub2
libkdumpfile
make-jpkg
makedumpfile
masking
misc-debs
nfs-utils
performance-diagnostics
ptools
Expand All @@ -26,4 +28,4 @@ recovery-environment
savedump
sdb
targetcli-fb
td-agent-prebuilt
virtualization
5 changes: 5 additions & 0 deletions package-lists/update/main.pkgs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ crash
drgn
grub2
libkdumpfile
linux-kernel-aws
linux-kernel-azure
linux-kernel-gcp
linux-kernel-generic
linux-kernel-oracle
makedumpfile
nfs-utils
python-rtslib-fb
Expand Down
15 changes: 8 additions & 7 deletions packages/adoptopenjdk/config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@
DEFAULT_PACKAGE_GIT_URL=none
PACKAGE_DEPENDENCIES="make-jpkg"

tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
_tarfile_sha256="733755fd649fad6ae91fc083f7e5a5a0b56410fb6ac1815cff29f744b128b1b1"
_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"

function prepare() {
logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb
Expand All @@ -29,15 +30,15 @@ function prepare() {
function fetch() {
logmust cd "$WORKDIR/"

local url="http://artifactory.delphix.com/artifactory"
local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile"

logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile"
logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256"
}

function build() {
logmust cd "$WORKDIR/"

logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<<y
logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<<y

logmust mv ./*deb "$WORKDIR/artifacts/"
#
Expand All @@ -49,7 +50,7 @@ function build() {
# packages, such as the app-gate, decide to fetch and install Java from
# the Linux-pkg bundle.
#
logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
logmust bash -c "echo $_jdk_path >'$WORKDIR/artifacts/JDK_PATH'"

echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
}
50 changes: 0 additions & 50 deletions packages/java8/config.sh

This file was deleted.

57 changes: 57 additions & 0 deletions packages/masking/config.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/bash
#
# Copyright 2021 Delphix
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# shellcheck disable=SC2034

DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/masking/dms-core-gate.git"
PACKAGE_DEPENDENCIES="adoptopenjdk"

function prepare() {
logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb
}

function build() {
export JAVA_HOME
JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"

logmust cd "$WORKDIR/repo"

#
# The "appliance-build-stage0" Jenkins job consumes this file,
# along with various other files (e.g. licensing metadata).
# Thus, if we don't generate it here, the Jenkins job that
# builds the appliance will fail.
#
# shellcheck disable=SC2016
logmust jq -n \
--arg h "$(git rev-parse HEAD)" \
--arg d "$(date --utc --iso-8601=seconds)" \
'{ "dms-core-gate" : { "git-hash" : $h, "date": $d }}' \
>"$WORKDIR/artifacts/metadata.json"

logmust ./gradlew --no-daemon --stacktrace \
-Porg.gradle.configureondemand=false \
-PenvironmentName=linuxappliance \
clean \
generateLicenseReport \
:dist:distDeb \
:dist:distLicenseReport

logmust rsync -av dist/build/distributions/ "$WORKDIR/artifacts/"
logmust cp -v \
dist/build/reports/dependency-license/* "$WORKDIR/artifacts/"
}
Loading

0 comments on commit 0c80c44

Please sign in to comment.