From 88dcfcde2e2979b407afef5eec533d62fc0d668e Mon Sep 17 00:00:00 2001
From: Pavel Zakharov <pavel.zakharov@delphix.com>
Date: Mon, 14 Dec 2020 16:53:47 -0500
Subject: [PATCH 01/11] DLPX-73310 linux-pkg: enable building from release tag
 (#130)

When using a release tag in DEFAULT_GIT_BRANCH the linux-package-mirror
link and the base url for fetching build dependencies are now properly
set up.
---
 README.md     | 12 +++++++-
 lib/common.sh | 83 +++++++++++++++++++++++++++++++++++++++++++--------
 setup.sh      | 14 +++++----
 3 files changed, 90 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index ec98f80..116552b 100644
--- a/README.md
+++ b/README.md
@@ -254,7 +254,9 @@ of some of the scripts defined above.
 
 * **DEFAULT_GIT_BRANCH**: The product branch that is being built or updated is
   typically stored in the file `branch.config`, however it can be overridden via
-  DEFAULT_GIT_BRANCH. The product branch is used in multiple instances. When
+  DEFAULT_GIT_BRANCH. It can either be set to a development branch, such as
+  "master" or "6.0/stage", or a release tag, such as "release/6.0.6.0".
+  The product branch is used in multiple instances. When
   running [setup.sh](#setupsh), it will determine what linux-package-mirror
   link to use when fetching packages from apt (although those links can be
   overridden via DELPHIX_PACKAGE_MIRROR_MAIN and
@@ -309,6 +311,14 @@ of some of the scripts defined above.
   dependencies produced by the developer Jenkins instance instead of the
   production one.
 
+* **DEPENDENCIES_BASE_URL**: When fetching artifacts from other linux-pkg
+  packages that are marked as dependencies of a package, we look for a
+  specific s3 path based on what product branch or version we are building for,
+  which is defined by DEFAULT_GIT_BRANCH. If DEPENDENCIES_BASE_URL is left
+  unset, then the path will be determined automatically. DEPENDENCIES_BASE_URL
+  is most useful when set to the input-artifacts of a previous appliance-build
+  run, i.e. "s3://.../input-artifacts/combined-packages/packages".
+
 ## Package Definition
 
 For each package built by this framework, there must be a file named
diff --git a/lib/common.sh b/lib/common.sh
index 0feba5a..5d3b450 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -26,7 +26,6 @@ export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle"
 # for testing purposes to use jenkins-ops.<developer> instead.
 #
 export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}"
-export _BASE_S3_URL="s3://snapshot-de-images/builds/${JENKINS_OPS_DIR}/devops-gate/master"
 
 export UBUNTU_DISTRIBUTION="bionic"
 
@@ -144,6 +143,11 @@ function determine_default_git_branch() {
 	export DEFAULT_GIT_BRANCH
 }
 
+function is_release_branch() {
+	check_env DEFAULT_GIT_BRANCH
+	[[ "$DEFAULT_GIT_BRANCH" == release/* ]]
+}
+
 function check_package_exists() {
 	local pkg="$1"
 
@@ -608,6 +612,69 @@ function default_revision() {
 	echo "delphix-$(date '+%Y.%m.%d.%H')"
 }
 
+function determine_dependencies_base_url() {
+	[[ -n "$DEPENDENCIES_BASE_URL" ]] && return
+
+	check_env DEFAULT_GIT_BRANCH
+
+	#
+	# The location of package artifacts depends on whether a release tag
+	# or a development branch is being built.
+	#
+	if is_release_branch; then
+		local version="${DEFAULT_GIT_BRANCH#release/}"
+		local url="s3://release-de-images/internal-artifacts/$version"
+		local suv
+		suv="$(aws s3 ls "$url/" | awk '{print $2}' | tr -d '/' | sort -V | tail -n 1)"
+		[[ -n "$suv" ]] || die "No artifacts found at $url"
+		DEPENDENCIES_BASE_URL="$url/$suv/input-artifacts/combined-packages/packages"
+	else
+		DEPENDENCIES_BASE_URL="s3://snapshot-de-images/builds/$JENKINS_OPS_DIR/devops-gate/master/linux-pkg/$DEFAULT_GIT_BRANCH/build-package"
+	fi
+
+	#
+	# Make sure the dependencies directory actually exists
+	#
+	logmust aws s3 ls "$DEPENDENCIES_BASE_URL"
+
+	export DEPENDENCIES_BASE_URL
+}
+
+function get_package_dependency_s3_url() {
+	local dep="$1"
+
+	logmust determine_dependencies_base_url
+
+	#
+	# The base url can be either pointing to:
+	# 1. Input artifacts from a previously built Delphix Appliance, or
+	# 2. A build-package/<package> directory that stores many builds of the
+	#    same package and a "latest" link that points to the latest build
+	#    for that package.
+	# In case 1 the artifacts are right there under the base url, whereas
+	# in case 2 we first need to dereference the "latest" link.
+	#
+	# The user can provide their own DEPENDENCIES_BASE_URL based on a
+	# previous Delphix Appliance build or have it determined automatically
+	# by determine_dependencies_base_url() based on the product branch
+	# being built.
+	#
+	if [[ "$DEPENDENCIES_BASE_URL" == */build-package ]]; then
+		local s3url="$DEPENDENCIES_BASE_URL/$dep/post-push"
+		(logmust aws s3 cp --only-show-errors "$s3url/latest" .) ||
+			die "Artifacts for dependency '$dep' missing." \
+				"Dependency must be built first."
+		logmust cat latest
+		local bucket="${DEPENDENCIES_BASE_URL#s3://}"
+		bucket=${bucket%%/*}
+		s3url="s3://$bucket/$(cat latest)"
+		logmust rm latest
+		_RET="$s3url"
+	else
+		_RET="$DEPENDENCIES_BASE_URL/$dep"
+	fi
+}
+
 #
 # Fetch artifacts from S3 for all packages listed in PACKAGE_DEPENDENCIES which
 # is defined in the package's config.
@@ -622,11 +689,6 @@ function fetch_dependencies() {
 		return
 	fi
 
-	local base_url="$_BASE_S3_URL/linux-pkg/$DEFAULT_GIT_BRANCH/build-package"
-
-	local bucket="${_BASE_S3_URL#s3://}"
-	bucket=${bucket%%/*}
-
 	local dep s3urlvar s3url
 	for dep in $PACKAGE_DEPENDENCIES; do
 		echo "Fetching artifacts for dependency '$dep' ..."
@@ -638,13 +700,8 @@ function fetch_dependencies() {
 				"externally"
 			echo "$s3urlvar=$s3url"
 		else
-			s3url="$base_url/$dep/post-push"
-			(logmust aws s3 cp --only-show-errors "$s3url/latest" .) ||
-				die "Artifacts for dependency '$dep' missing." \
-					"Dependency must be built first."
-			logmust cat latest
-			s3url="s3://$bucket/$(cat latest)"
-			logmust rm latest
+			logmust get_package_dependency_s3_url "$dep"
+			s3url="$_RET"
 		fi
 		[[ "$s3url" != */ ]] && s3url="$s3url/"
 		logmust mkdir "$dep"
diff --git a/setup.sh b/setup.sh
index 2d0b232..0319263 100755
--- a/setup.sh
+++ b/setup.sh
@@ -31,11 +31,15 @@ configure_apt_sources() {
 
 	if [[ -z "$primary_url" ]] || [[ -z "$secondary_url" ]]; then
 		local latest_url="http://linux-package-mirror.delphix.com/"
-		latest_url+="${DEFAULT_GIT_BRANCH}/latest/"
-		package_mirror_url=$(curl -LfSs -o /dev/null -w '%{url_effective}' \
-			"$latest_url" || die "Could not curl $latest_url")
-		# Remove trailing slash, if present.
-		package_mirror_url="${package_mirror_url%/}"
+		if is_release_branch; then
+			package_mirror_url="${latest_url}${DEFAULT_GIT_BRANCH}"
+		else
+			latest_url+="${DEFAULT_GIT_BRANCH}/latest/"
+			package_mirror_url=$(curl -LfSs -o /dev/null -w '%{url_effective}' \
+				"$latest_url" || die "Could not curl $latest_url")
+			# Remove trailing slash, if present.
+			package_mirror_url="${package_mirror_url%/}"
+		fi
 		[[ -z "$primary_url" ]] && primary_url="${package_mirror_url}/ubuntu"
 		[[ -z "$secondary_url" ]] && secondary_url="${package_mirror_url}/ppas"
 	fi

From 219037ad103fd0ef1f1a981da4fbed258982ad18 Mon Sep 17 00:00:00 2001
From: Pavel Zakharov <pavel.zakharov@delphix.com>
Date: Mon, 11 Jan 2021 11:27:51 -0500
Subject: [PATCH 02/11] DLPX-73610 pin unzip package to version 6.0-21ubuntu1
 (#132)

---
 lib/common.sh                        | 21 +++++++-
 package-lists/build/main.pkgs        |  2 +-
 packages/adoptopenjdk/config.sh      | 15 +++---
 packages/java8/config.sh             | 50 -----------------
 packages/misc-debs/config.sh         | 81 ++++++++++++++++++++++++++++
 packages/td-agent-prebuilt/config.sh | 33 ------------
 6 files changed, 110 insertions(+), 92 deletions(-)
 delete mode 100644 packages/java8/config.sh
 create mode 100644 packages/misc-debs/config.sh
 delete mode 100644 packages/td-agent-prebuilt/config.sh

diff --git a/lib/common.sh b/lib/common.sh
index 5d3b450..b9bd29e 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -1116,7 +1116,26 @@ function fetch_kernel_from_artifactory() {
 	url="$url/linux-pkg/linux-prebuilt/${artifactory_deb}"
 
 	logmust cd "$WORKDIR/artifacts"
-	logmust wget -nv "$url"
+	fetch_file_from_artifactory "$url"
+}
+
+function fetch_file_from_artifactory() {
+	local url="$1"
+	local sha256_expected="$2"
+	local file sha256_actual
+
+	[[ -n "$url" ]] || die "url argument is missing."
+	file="$(basename "$url")"
+
+	logmust wget -nv "$url" -O "$file"
+
+	if [[ -n "$sha256_expected" ]]; then
+		sha256_actual="$(sha256sum "$file" | awk '{print $1}')"
+		if [[ "$sha256_expected" != "$sha256_actual" ]]; then
+			die "SHA256 mismatch. Expected: $sha256_expected," \
+				"Actual: $sha256_actual"
+		fi
+	fi
 }
 
 #
diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs
index 28bffca..79bb0dd 100644
--- a/package-lists/build/main.pkgs
+++ b/package-lists/build/main.pkgs
@@ -18,6 +18,7 @@ grub2
 libkdumpfile
 make-jpkg
 makedumpfile
+misc-debs
 nfs-utils
 performance-diagnostics
 ptools
@@ -26,4 +27,3 @@ recovery-environment
 savedump
 sdb
 targetcli-fb
-td-agent-prebuilt
diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh
index eb14a27..13ea440 100755
--- a/packages/adoptopenjdk/config.sh
+++ b/packages/adoptopenjdk/config.sh
@@ -19,8 +19,9 @@
 DEFAULT_PACKAGE_GIT_URL=none
 PACKAGE_DEPENDENCIES="make-jpkg"
 
-tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
-jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
+_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
+_tarfile_sha256="733755fd649fad6ae91fc083f7e5a5a0b56410fb6ac1815cff29f744b128b1b1"
+_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
 
 function prepare() {
 	logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb
@@ -29,15 +30,15 @@ function prepare() {
 function fetch() {
 	logmust cd "$WORKDIR/"
 
-	local url="http://artifactory.delphix.com/artifactory"
+	local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile"
 
-	logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile"
+	logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256"
 }
 
 function build() {
 	logmust cd "$WORKDIR/"
 
-	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<<y
+	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<<y
 
 	logmust mv ./*deb "$WORKDIR/artifacts/"
 	#
@@ -49,7 +50,7 @@ function build() {
 	# packages, such as the app-gate, decide to fetch and install Java from
 	# the Linux-pkg bundle.
 	#
-	logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
+	logmust bash -c "echo $_jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
 
-	echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
+	echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
 }
diff --git a/packages/java8/config.sh b/packages/java8/config.sh
deleted file mode 100644
index 5d03bae..0000000
--- a/packages/java8/config.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2019 Delphix
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# shellcheck disable=SC2034
-
-DEFAULT_PACKAGE_GIT_URL=none
-
-tarfile="jdk-8u171-linux-x64.tar.gz"
-jdk_path="/usr/lib/jvm/oracle-java8-jdk-amd64"
-
-function prepare() {
-	logmust install_pkgs java-package
-}
-
-function fetch() {
-	logmust cd "$WORKDIR"
-	local url="http://artifactory.delphix.com/artifactory"
-	logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile"
-}
-
-function build() {
-	logmust cd "$WORKDIR"
-	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<<y
-	logmust mv ./*deb "$WORKDIR/artifacts/"
-	#
-	# Store the install path of the JDK in a file so that the users of this
-	# Java package know where to look. This is especially useful for
-	# other linux-pkg packages that have a build dependency on this
-	# particular version of Java, as they don't have to hardcode the
-	# path in their build definition. This would also be useful if external
-	# packages, such as the app-gate, decide to fetch and install Java from
-	# the Linux-pkg bundle.
-	#
-	logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
-
-	echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
-}
diff --git a/packages/misc-debs/config.sh b/packages/misc-debs/config.sh
new file mode 100644
index 0000000..5f08ce4
--- /dev/null
+++ b/packages/misc-debs/config.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL=none
+SKIP_COPYRIGHTS_CHECK=true
+
+#
+# IMPORTANT NOTE
+# --------------
+#
+# Debian packages (debs) that are not built from source by linux-pkg can be
+# added to this "meta-package". As a general rule, pre-built debs should only
+# be added here when they have been fetched from a trusted third-party
+# package archive.
+#
+# Here are some valid reasons for adding new debs here:
+# - There are bugs with a recent version of a package provided by Ubuntu and
+#   we want to pin an older version of that package.
+# - Ubuntu provides a version of a package that is too old, and the package's
+#   maintainers provide a more recent version of the package. Note that in this
+#   case, you may also look into adding the maintainer's archive to the
+#   linux-package-mirror PPAs list.
+#
+# To add a new deb here, upload that deb to the linux-pkg/misc-debs directory
+# in artifcatory and note the deb's SHA256. Be explicit on where this deb
+# was fetched from and why it was added to this list.
+#
+# When removing debs from this list, you should not remove them from artifactory
+# as they would used when rebuilding older releases.
+#
+
+function fetch() {
+	logmust cd "$WORKDIR/artifacts"
+
+	#
+	# Note about the debs being fetched:
+	# - td-agent was built by the "td-agent" linux-pkg package, but it
+	#   now fails to build due to broken third party dependencies. See
+	#   DLPX-69338 and DLPX-68211.
+	# - unzip was added as a temporary workaround to DLPX-73555.
+	#
+	local debs=(
+		"td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb 84dfa2436039ff2a6312484bd7295ebaf570b5f59f100380b57e68b4800855c4"
+		"unzip_6.0-21ubuntu1_amd64.deb d46069c369ce88c8dd91c52abb8de8d6053606748ef18b3b9bc290fdd8ad2953"
+	)
+
+	local url="http://artifactory.delphix.com/artifactory/linux-pkg/misc-debs"
+
+	echo "Fetched debs:" >BUILD_INFO
+	local entry
+	for entry in "${debs[@]}"; do
+		local deb sha256
+		deb=$(echo "$entry" | awk '{print $1}')
+		sha256=$(echo "$entry" | awk '{print $2}')
+		[[ -n "$deb" && -n "$sha256" ]] || die "Invalid entry '$entry'"
+
+		logmust fetch_file_from_artifactory "$url/$deb" "$sha256"
+
+		echo "$entry" >>BUILD_INFO
+	done
+}
+
+function build() {
+	return
+	# Nothing to do, all the logic is done in fetch().
+}
diff --git a/packages/td-agent-prebuilt/config.sh b/packages/td-agent-prebuilt/config.sh
deleted file mode 100644
index 8e5e215..0000000
--- a/packages/td-agent-prebuilt/config.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2019 Delphix
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# shellcheck disable=SC2034
-
-DEFAULT_PACKAGE_GIT_URL=none
-SKIP_COPYRIGHTS_CHECK=true
-
-function fetch() {
-	logmust cd "$WORKDIR/artifacts"
-	local package="td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb"
-	local url="http://artifactory.delphix.com/artifactory"
-
-	logmust wget -nv "$url/linux-pkg/td-agent/$package" -O "$package"
-}
-
-function build() {
-	return
-	# Nothing to do. See the td-agent package config for the actual build.
-}

From 91a3510f14870586363ca7d2838ae49a2af4fc54 Mon Sep 17 00:00:00 2001
From: Pavel Zakharov <pavel.zakharov@delphix.com>
Date: Wed, 13 Jan 2021 17:22:34 -0500
Subject: [PATCH 03/11] DLPX-73651 [Backport of DLPX-73610 to 6.0.7.0] pin
 unzip package to version 6.0-21ubuntu1 (#134)

---
 lib/common.sh                        | 21 +++++++-
 package-lists/build/main.pkgs        |  2 +-
 packages/adoptopenjdk/config.sh      | 15 +++---
 packages/java8/config.sh             | 50 -----------------
 packages/misc-debs/config.sh         | 81 ++++++++++++++++++++++++++++
 packages/td-agent-prebuilt/config.sh | 33 ------------
 6 files changed, 110 insertions(+), 92 deletions(-)
 delete mode 100644 packages/java8/config.sh
 create mode 100644 packages/misc-debs/config.sh
 delete mode 100644 packages/td-agent-prebuilt/config.sh

diff --git a/lib/common.sh b/lib/common.sh
index daa64a1..59259dc 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -1116,7 +1116,26 @@ function fetch_kernel_from_artifactory() {
 	url="$url/linux-pkg/linux-prebuilt/${artifactory_deb}"
 
 	logmust cd "$WORKDIR/artifacts"
-	logmust wget -nv "$url"
+	fetch_file_from_artifactory "$url"
+}
+
+function fetch_file_from_artifactory() {
+	local url="$1"
+	local sha256_expected="$2"
+	local file sha256_actual
+
+	[[ -n "$url" ]] || die "url argument is missing."
+	file="$(basename "$url")"
+
+	logmust wget -nv "$url" -O "$file"
+
+	if [[ -n "$sha256_expected" ]]; then
+		sha256_actual="$(sha256sum "$file" | awk '{print $1}')"
+		if [[ "$sha256_expected" != "$sha256_actual" ]]; then
+			die "SHA256 mismatch. Expected: $sha256_expected," \
+				"Actual: $sha256_actual"
+		fi
+	fi
 }
 
 #
diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs
index 28bffca..79bb0dd 100644
--- a/package-lists/build/main.pkgs
+++ b/package-lists/build/main.pkgs
@@ -18,6 +18,7 @@ grub2
 libkdumpfile
 make-jpkg
 makedumpfile
+misc-debs
 nfs-utils
 performance-diagnostics
 ptools
@@ -26,4 +27,3 @@ recovery-environment
 savedump
 sdb
 targetcli-fb
-td-agent-prebuilt
diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh
index eb14a27..13ea440 100755
--- a/packages/adoptopenjdk/config.sh
+++ b/packages/adoptopenjdk/config.sh
@@ -19,8 +19,9 @@
 DEFAULT_PACKAGE_GIT_URL=none
 PACKAGE_DEPENDENCIES="make-jpkg"
 
-tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
-jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
+_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
+_tarfile_sha256="733755fd649fad6ae91fc083f7e5a5a0b56410fb6ac1815cff29f744b128b1b1"
+_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
 
 function prepare() {
 	logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb
@@ -29,15 +30,15 @@ function prepare() {
 function fetch() {
 	logmust cd "$WORKDIR/"
 
-	local url="http://artifactory.delphix.com/artifactory"
+	local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile"
 
-	logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile"
+	logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256"
 }
 
 function build() {
 	logmust cd "$WORKDIR/"
 
-	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<<y
+	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<<y
 
 	logmust mv ./*deb "$WORKDIR/artifacts/"
 	#
@@ -49,7 +50,7 @@ function build() {
 	# packages, such as the app-gate, decide to fetch and install Java from
 	# the Linux-pkg bundle.
 	#
-	logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
+	logmust bash -c "echo $_jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
 
-	echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
+	echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
 }
diff --git a/packages/java8/config.sh b/packages/java8/config.sh
deleted file mode 100644
index 5d03bae..0000000
--- a/packages/java8/config.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2019 Delphix
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# shellcheck disable=SC2034
-
-DEFAULT_PACKAGE_GIT_URL=none
-
-tarfile="jdk-8u171-linux-x64.tar.gz"
-jdk_path="/usr/lib/jvm/oracle-java8-jdk-amd64"
-
-function prepare() {
-	logmust install_pkgs java-package
-}
-
-function fetch() {
-	logmust cd "$WORKDIR"
-	local url="http://artifactory.delphix.com/artifactory"
-	logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile"
-}
-
-function build() {
-	logmust cd "$WORKDIR"
-	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<<y
-	logmust mv ./*deb "$WORKDIR/artifacts/"
-	#
-	# Store the install path of the JDK in a file so that the users of this
-	# Java package know where to look. This is especially useful for
-	# other linux-pkg packages that have a build dependency on this
-	# particular version of Java, as they don't have to hardcode the
-	# path in their build definition. This would also be useful if external
-	# packages, such as the app-gate, decide to fetch and install Java from
-	# the Linux-pkg bundle.
-	#
-	logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
-
-	echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
-}
diff --git a/packages/misc-debs/config.sh b/packages/misc-debs/config.sh
new file mode 100644
index 0000000..5f08ce4
--- /dev/null
+++ b/packages/misc-debs/config.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL=none
+SKIP_COPYRIGHTS_CHECK=true
+
+#
+# IMPORTANT NOTE
+# --------------
+#
+# Debian packages (debs) that are not built from source by linux-pkg can be
+# added to this "meta-package". As a general rule, pre-built debs should only
+# be added here when they have been fetched from a trusted third-party
+# package archive.
+#
+# Here are some valid reasons for adding new debs here:
+# - There are bugs with a recent version of a package provided by Ubuntu and
+#   we want to pin an older version of that package.
+# - Ubuntu provides a version of a package that is too old, and the package's
+#   maintainers provide a more recent version of the package. Note that in this
+#   case, you may also look into adding the maintainer's archive to the
+#   linux-package-mirror PPAs list.
+#
+# To add a new deb here, upload that deb to the linux-pkg/misc-debs directory
+# in artifcatory and note the deb's SHA256. Be explicit on where this deb
+# was fetched from and why it was added to this list.
+#
+# When removing debs from this list, you should not remove them from artifactory
+# as they would used when rebuilding older releases.
+#
+
+function fetch() {
+	logmust cd "$WORKDIR/artifacts"
+
+	#
+	# Note about the debs being fetched:
+	# - td-agent was built by the "td-agent" linux-pkg package, but it
+	#   now fails to build due to broken third party dependencies. See
+	#   DLPX-69338 and DLPX-68211.
+	# - unzip was added as a temporary workaround to DLPX-73555.
+	#
+	local debs=(
+		"td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb 84dfa2436039ff2a6312484bd7295ebaf570b5f59f100380b57e68b4800855c4"
+		"unzip_6.0-21ubuntu1_amd64.deb d46069c369ce88c8dd91c52abb8de8d6053606748ef18b3b9bc290fdd8ad2953"
+	)
+
+	local url="http://artifactory.delphix.com/artifactory/linux-pkg/misc-debs"
+
+	echo "Fetched debs:" >BUILD_INFO
+	local entry
+	for entry in "${debs[@]}"; do
+		local deb sha256
+		deb=$(echo "$entry" | awk '{print $1}')
+		sha256=$(echo "$entry" | awk '{print $2}')
+		[[ -n "$deb" && -n "$sha256" ]] || die "Invalid entry '$entry'"
+
+		logmust fetch_file_from_artifactory "$url/$deb" "$sha256"
+
+		echo "$entry" >>BUILD_INFO
+	done
+}
+
+function build() {
+	return
+	# Nothing to do, all the logic is done in fetch().
+}
diff --git a/packages/td-agent-prebuilt/config.sh b/packages/td-agent-prebuilt/config.sh
deleted file mode 100644
index 8e5e215..0000000
--- a/packages/td-agent-prebuilt/config.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2019 Delphix
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# shellcheck disable=SC2034
-
-DEFAULT_PACKAGE_GIT_URL=none
-SKIP_COPYRIGHTS_CHECK=true
-
-function fetch() {
-	logmust cd "$WORKDIR/artifacts"
-	local package="td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb"
-	local url="http://artifactory.delphix.com/artifactory"
-
-	logmust wget -nv "$url/linux-pkg/td-agent/$package" -O "$package"
-}
-
-function build() {
-	return
-	# Nothing to do. See the td-agent package config for the actual build.
-}

From 2c3d6ad180ad353807f94403a0a9851b821cf697 Mon Sep 17 00:00:00 2001
From: Pavel Zakharov <pavel.zakharov@delphix.com>
Date: Tue, 19 Jan 2021 17:53:10 -0500
Subject: [PATCH 04/11] DLPX-73657 linux-pkg auto-update: git tags for hwe
 kernel now has the same format as other kernel flavours (#133)

---
 default-package-config.sh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/default-package-config.sh b/default-package-config.sh
index 3a95072..88f3490 100644
--- a/default-package-config.sh
+++ b/default-package-config.sh
@@ -220,24 +220,24 @@ function kernel_update_upstream() {
 	# Note that "generic" (used mainly ESX) is a special
 	# case as we are currently using the HWE kernel image.
 	#
-	local tag_prefix
+	local tag_prefix_flavour
 	if [[ "${platform}" == generic ]] &&
 		[[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then
-		tag_prefix="Ubuntu-hwe-${kernel_version}-${abinum}"
+		tag_prefix_flavour="Ubuntu-hwe"
 	elif [[ "${platform}" == aws ]] ||
 		[[ "${platform}" == azure ]] ||
 		[[ "${platform}" == gcp ]] ||
 		[[ "${platform}" == oracle ]]; then
-
-		local kvers_major kvers_minor short_kvers
-		kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
-		kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
-		short_kvers="${kvers_major}.${kvers_minor}"
-
-		tag_prefix="Ubuntu-${platform}-${short_kvers}-${kernel_version}-${abinum}"
+		tag_prefix_flavour="Ubuntu-${platform}"
 	else
 		die "assertion: unexpected platform: ${platform}"
 	fi
+
+	local tag_prefix kvers_major kvers_minor short_kvers
+	kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
+	kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
+	short_kvers="${kvers_major}.${kvers_minor}"
+	tag_prefix="${tag_prefix_flavour}-${short_kvers}-${kernel_version}-${abinum}"
 	echo "note: upstream tag prefix used: ${tag_prefix}"
 
 	#

From 517cfb98f3335a07a814521d21dd58051c315814 Mon Sep 17 00:00:00 2001
From: Pavel Zakharov <pavel.zakharov@delphix.com>
Date: Wed, 20 Jan 2021 15:47:35 -0500
Subject: [PATCH 05/11] DLPX-73763 Fix kernel build and auto-sync logic (#135)

---
 default-package-config.sh | 42 +++++++++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/default-package-config.sh b/default-package-config.sh
index 88f3490..efb0bcd 100644
--- a/default-package-config.sh
+++ b/default-package-config.sh
@@ -61,7 +61,8 @@ function kernel_build() {
 	#       debian/rules command, the rightmost declaration
 	#       is the one that is actually used.
 	#
-	local debian_rules_extra_args="$2"
+	shift
+	local debian_rules_extra_args=("$@")
 
 	logmust cd "$WORKDIR/repo"
 
@@ -91,7 +92,7 @@ function kernel_build() {
 	#
 	local canonical_abinum delphix_abinum kernel_release kernel_version
 	canonical_abinum=$(fakeroot debian/rules printenv | grep -E '^abinum ' | cut -d= -f2 | tr -d '[:space:]')
-	delphix_abinum="dlpx-$(date -u +"%Y%m%dt%H%M%S")-$(git rev-parse --short HEAD)-${canonical_abinum}"
+	delphix_abinum="${canonical_abinum}-$(date -u +"dx%Y%m%d%H")-$(git rev-parse --short HEAD)"
 	kernel_release=$(fakeroot debian/rules printenv | grep -E '^release ' | cut -d= -f2 | tr -d '[:space:]')
 
 	#
@@ -113,20 +114,35 @@ function kernel_build() {
 	# disable_d_i=true
 	#   This prevents udeb packages from being built as they are
 	#   not consumed by the Delphix Appliance.
-	#
-	local debian_rules_args="skipdbg=false uefi_signed=false disable_d_i=true flavours=$platform abinum=${delphix_abinum} ${debian_rules_extra_args}"
+	# do_dkms_*=false
+	#   This disables the build of various out-of-tree kernel modules
+	#   that we do not use in our product or that we provide separately.
+	#
+	local debian_rules_args=(
+		"skipdbg=false"
+		"uefi_signed=false"
+		"disable_d_i=true"
+		"do_zfs=false"
+		"do_dkms_nvidia=false"
+		"do_dkms_nvidia_server=false"
+		"do_dkms_vbox=false"
+		"do_dkms_wireguard=false"
+		"flavours=$platform"
+		"abinum=${delphix_abinum}"
+	)
+	debian_rules_args+=("${debian_rules_extra_args[@]}")
 
 	#
 	# Clean up everything generated so far and recreate the
 	# final control file with the arguments that we want.
 	#
-	logmust fakeroot debian/rules clean ${debian_rules_args}
+	logmust fakeroot debian/rules clean "${debian_rules_args[@]}"
 
 	#
 	# Print the environment configuration solely for
 	# debugging purposes.
 	#
-	logmust fakeroot debian/rules printenv ${debian_rules_args}
+	logmust fakeroot debian/rules printenv "${debian_rules_args[@]}"
 
 	#
 	# The default value of the tool argument for mk-build-deps
@@ -139,7 +155,7 @@ function kernel_build() {
 	local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes"
 	logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}"
 
-	logmust fakeroot debian/rules "binary" ${debian_rules_args}
+	logmust fakeroot debian/rules "binary" "${debian_rules_args[@]}"
 
 	logmust cd "$WORKDIR"
 	logmust mv ./*deb "artifacts/"
@@ -274,8 +290,18 @@ function kernel_update_upstream() {
 
 	logmust git fetch upstream "+refs/tags/${upstream_tag}:refs/tags/${upstream_tag}"
 
+	#
+	# Note that we add '^{}' at the end to dereference the tag recursively
+	# until it arrives to an actual commit. This is needed in case the
+	# tag points to an anotated tag object which contains extra information
+	# such as a PGP signature. That annoted tag will in turn reference the
+	# actual commit, which will be returned when appending ^{}.
+	# upstream-HEAD will be pointing to the commit directly rather than the
+	# annoted tag object, so if we want to compare the two we need to query
+	# for the dereferenced commit. See 'git help gitrevisions' for more info.
+	#
 	local upstream_tag_commit
-	upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}")" ||
+	upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}^{}")" ||
 		die "couldn't get commit of tag ${upstream_tag}"
 	echo "note: upstream tag: ${upstream_tag}, commit ${upstream_tag_commit}"
 

From c155865390377d24c22f6e12d18321d888ff4457 Mon Sep 17 00:00:00 2001
From: Pavel Zakharov <pavel.zakharov@delphix.com>
Date: Fri, 22 Jan 2021 17:26:13 -0500
Subject: [PATCH 06/11] DLPX-73791 Use linux kernel packages built in-house
 (#136)

---
 lib/common.sh                  | 2 +-
 package-lists/update/main.pkgs | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/common.sh b/lib/common.sh
index b9bd29e..5153767 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -35,7 +35,7 @@ export UBUNTU_DISTRIBUTION="bionic"
 #  2. "archive": dowloading from apt
 #  3. "prebuilt": pre-built kernel stored in artifactory
 #
-export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="archive"
+export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="delphix"
 
 # shellcheck disable=SC2086
 function enable_colors() {
diff --git a/package-lists/update/main.pkgs b/package-lists/update/main.pkgs
index 73ff7c3..b2654f9 100644
--- a/package-lists/update/main.pkgs
+++ b/package-lists/update/main.pkgs
@@ -17,6 +17,11 @@ crash
 drgn
 grub2
 libkdumpfile
+linux-kernel-aws
+linux-kernel-azure
+linux-kernel-gcp
+linux-kernel-generic
+linux-kernel-oracle
 makedumpfile
 nfs-utils
 python-rtslib-fb

From 9c8192c0e465afb8d0a8c4d96a8038d1dffe1018 Mon Sep 17 00:00:00 2001
From: Raul Santelices <raul.santelices@delphix.com>
Date: Tue, 16 Feb 2021 11:07:19 -0500
Subject: [PATCH 07/11] Fix branch.config

---
 branch.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/branch.config b/branch.config
index 705e477..ab48b35 100644
--- a/branch.config
+++ b/branch.config
@@ -11,4 +11,4 @@
 # branching.
 #
 
-DEFAULT_GIT_BRANCH="6.0/stage"
+DEFAULT_GIT_BRANCH="6.0/release"

From 42d16d7ed4b88fdb04e0b3c6cc79b4f4e5cc1d45 Mon Sep 17 00:00:00 2001
From: Prakash Surya <prakash.surya@delphix.com>
Date: Tue, 23 Mar 2021 13:28:40 -0700
Subject: [PATCH 08/11] TOOL-11187 leverage linux-pkg to build delphix-masking
 package (#139) (#146)

---
 lib/common.sh              |  2 +-
 packages/masking/config.sh | 57 ++++++++++++++++++++++++++++++++++++++
 query-packages.sh          |  6 ++++
 setup.sh                   |  4 ++-
 4 files changed, 67 insertions(+), 2 deletions(-)
 create mode 100644 packages/masking/config.sh

diff --git a/lib/common.sh b/lib/common.sh
index 5153767..caf4fd8 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -89,7 +89,7 @@ function die() {
 }
 
 function logmust() {
-	echo Running: "$@"
+	[[ "$LOGGING" == "false" ]] || echo Running: "$@" >&2
 	"$@" || die "failed command '$*'"
 }
 
diff --git a/packages/masking/config.sh b/packages/masking/config.sh
new file mode 100644
index 0000000..a4daf38
--- /dev/null
+++ b/packages/masking/config.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/masking/dms-core-gate.git"
+PACKAGE_DEPENDENCIES="adoptopenjdk"
+
+function prepare() {
+	logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb
+}
+
+function build() {
+	export JAVA_HOME
+	JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
+		die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"
+
+	logmust cd "$WORKDIR/repo"
+
+	#
+	# The "appliance-build-stage0" Jenkins job consumes this file,
+	# along with various other files (e.g. licensing metadata).
+	# Thus, if we don't generate it here, the Jenkins job that
+	# builds the appliance will fail.
+	#
+	# shellcheck disable=SC2016
+	logmust jq -n \
+		--arg h "$(git rev-parse HEAD)" \
+		--arg d "$(date --utc --iso-8601=seconds)" \
+		'{ "dms-core-gate" : { "git-hash" : $h, "date": $d }}' \
+		>"$WORKDIR/artifacts/metadata.json"
+
+	logmust ./gradlew --no-daemon --stacktrace \
+		-Porg.gradle.configureondemand=false \
+		-PenvironmentName=linuxappliance \
+		clean \
+		generateLicenseReport \
+		:dist:distDeb \
+		:dist:distLicenseReport
+
+	logmust rsync -av dist/build/distributions/ "$WORKDIR/artifacts/"
+	logmust cp -v \
+		dist/build/reports/dependency-license/* "$WORKDIR/artifacts/"
+}
diff --git a/query-packages.sh b/query-packages.sh
index b72f81a..8cab51f 100755
--- a/query-packages.sh
+++ b/query-packages.sh
@@ -19,6 +19,12 @@ cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 TOP="$PWD"
 source "$TOP/lib/common.sh"
 
+#
+# Disable "logmust" output as that is unnecessary here, and actually
+# trips up some of our testing (e.g. verify-query-packages.sh).
+#
+export LOGGING=false
+
 ALL_OUTPUT_FIELDS=(name git-url dependencies can-update)
 
 function usage() {
diff --git a/setup.sh b/setup.sh
index 0319263..5fd99dd 100755
--- a/setup.sh
+++ b/setup.sh
@@ -93,12 +93,14 @@ logmust sudo apt-get update
 #   build dependencies from a control file.
 # - install_shfmt and shellcheck are needed for - make check - to be able to
 #   make sure style checks are fine.
+# - jq is used to generate a JSON formatted metadata file by some packages.
 #
 logmust install_pkgs \
 	debhelper \
 	devscripts \
 	equivs \
-	shellcheck
+	shellcheck \
+	jq
 
 logmust install_shfmt
 

From 148670ad7efca069eb03743aa0ab5ee4a6a9a962 Mon Sep 17 00:00:00 2001
From: Prakash Surya <prakash.surya@delphix.com>
Date: Tue, 23 Mar 2021 13:28:54 -0700
Subject: [PATCH 09/11] TOOL-11245 leverage linux-pkg to build
 delphix-virtualization package (#144) (#148)

---
 lib/common.sh                     |   2 +-
 packages/virtualization/config.sh | 121 ++++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+), 1 deletion(-)
 create mode 100644 packages/virtualization/config.sh

diff --git a/lib/common.sh b/lib/common.sh
index caf4fd8..9095307 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -470,7 +470,7 @@ function install_pkgs() {
 		echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
 			"apt-get install -y $*"
 		sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
-			-y "$@" && return
+			-y --allow-downgrades "$@" && return
 		echo "apt-get install failed, retrying."
 		sleep 10
 	done
diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh
new file mode 100644
index 0000000..20c01f8
--- /dev/null
+++ b/packages/virtualization/config.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/dlpx-app-gate.git"
+PACKAGE_DEPENDENCIES="adoptopenjdk crypt-blowfish misc-debs"
+
+function prepare() {
+	logmust install_pkgs \
+		ant \
+		gcc \
+		libcairo2 \
+		libcurl4-openssl-dev \
+		libjbig0 \
+		libnss3-dev \
+		libnss3-dbg \
+		libnss3-tools \
+		libpam0g-dev \
+		libpixman-1-0 \
+		libssl-dev \
+		libtiff5 \
+		libxcb-render0 \
+		libxcb-shm0 \
+		python-jira \
+		python-requests \
+		rsync
+
+	logmust install_pkgs \
+		"$DEPDIR"/adoptopenjdk/*.deb \
+		"$DEPDIR"/crypt-blowfish/*.deb \
+		"$DEPDIR"/misc-debs/unzip_6.0-21ubuntu1_amd64.deb
+}
+
+function build() {
+	export JAVA_HOME
+	JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
+		die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"
+
+	export LANG
+	LANG=en_US.UTF-8
+
+	logmust cd "$WORKDIR/repo"
+
+	#
+	# The "appliance-build-stage0" Jenkins job consumes this file,
+	# along with various other files (e.g. licensing metadata).
+	# Thus, if we don't generate it here, the Jenkins job that
+	# builds the appliance will fail.
+	#
+	# shellcheck disable=SC2016
+	logmust jq -n \
+		--arg h "$(git rev-parse HEAD)" \
+		--arg d "$(date --utc --iso-8601=seconds)" \
+		'{ "dlpx-app-gate" : { "git-hash" : $h, "date": $d }}' \
+		>"$WORKDIR/artifacts/metadata.json"
+
+	#
+	# Build the virtualization package
+	#
+	logmust cd "$WORKDIR/repo/appliance"
+	if [[ -n "$DELPHIX_RELEASE_VERSION" ]]; then
+		logmust ant -Ddockerize=true -DbuildJni=true \
+			-DhotfixGenDlpxVersion="$DELPHIX_RELEASE_VERSION" \
+			all package
+	else
+		logmust ant -Ddockerize=true -DbuildJni=true all package
+	fi
+
+	#
+	# Publish the virtualization package artifacts
+	#
+	logmust cd "$WORKDIR/repo/appliance"
+	logmust rsync -av packaging/build/distributions/ "$WORKDIR/artifacts/"
+	logmust cp -vr \
+		bin/out/common/com.delphix.common/uem/tars \
+		"$WORKDIR/artifacts/hostchecker2"
+	logmust cp -v \
+		server/api/build/api/json-schemas/delphix.json \
+		"$WORKDIR/artifacts"
+	logmust cp -v \
+		dist/server/opt/delphix/client/etc/api.ini \
+		"$WORKDIR/artifacts"
+	logmust cp -v \
+		packaging/build/reports/dependency-license/* \
+		"$WORKDIR/artifacts/"
+
+	#
+	# Build the "toolkit-devkit" artifacts
+	#
+	logmust cd "$WORKDIR/repo/appliance/toolkit"
+	if [[ -n "$DELPHIX_RELEASE_VERSION" ]]; then
+		logmust ant \
+			-Dversion.number="$DELPHIX_RELEASE_VERSION" \
+			toolkit-devkit
+	else
+		logmust ant \
+			"-Dversion.number=$(date --utc +%Y-%m-%d-%H-%m)" \
+			toolkit-devkit
+	fi
+
+	#
+	# Publish the "toolkit-devkit" artifacts
+	#
+	logmust cd "$WORKDIR/repo/appliance"
+	logmust mkdir -p "$WORKDIR/artifacts/hostchecker2"
+	logmust cp -v toolkit/toolkit-devkit.tar "$WORKDIR/artifacts"
+}

From 69b94e7886dae0d2ae3cce75789f8fb2c71730d0 Mon Sep 17 00:00:00 2001
From: Prakash Surya <prakash.surya@delphix.com>
Date: Tue, 23 Mar 2021 14:09:48 -0700
Subject: [PATCH 10/11] TOOL-11187 leverage linux-pkg to build delphix-masking
 package (#141) (#147)

---
 package-lists/build/main.pkgs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs
index 79bb0dd..937b535 100644
--- a/package-lists/build/main.pkgs
+++ b/package-lists/build/main.pkgs
@@ -18,6 +18,7 @@ grub2
 libkdumpfile
 make-jpkg
 makedumpfile
+masking
 misc-debs
 nfs-utils
 performance-diagnostics

From 8584c9845b8f473a86aa4ea074cd67ad81c7681c Mon Sep 17 00:00:00 2001
From: Prakash Surya <prakash.surya@delphix.com>
Date: Tue, 23 Mar 2021 14:09:57 -0700
Subject: [PATCH 11/11] TOOL-11245 leverage linux-pkg to build
 delphix-virtualization package (#145) (#149)

---
 package-lists/build/main.pkgs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs
index 937b535..fc00dd0 100644
--- a/package-lists/build/main.pkgs
+++ b/package-lists/build/main.pkgs
@@ -28,3 +28,4 @@ recovery-environment
 savedump
 sdb
 targetcli-fb
+virtualization