From 7e0ca5302c01b64d24b45af570f6dafc91e18f70 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Mon, 30 Sep 2019 09:45:28 -0700 Subject: [PATCH 01/39] Fix drgn build failures caused by building elfutil from scratch Recently drgn started builting the latest elfutils as part of its building process. Ensure that the prerequisites for that library are installed on the system before attempting to build drgn. For reference the commits that caused this are the following: 1cedca8ff4e39207aa5b0a1a34793dcbdb1f025f 6a13d74c0c45881c7e26b3c16a2d76413c740641 --- packages/drgn/config.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index c15ac91..e66a052 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -26,13 +26,21 @@ function prepare() { logmust install_pkgs \ autoconf \ automake \ + bison \ + flex \ + gawk \ git \ + gcc \ + libbz2-dev \ libdw-dev \ libelf-dev \ + liblzma-dev \ libtool \ + make \ pkg-config \ python3-distutils \ - python3.6-dev + python3.6-dev \ + zlib1g-dev } function build() { From 9439ed9da73ca4e5215c97bdd04f4911d0580c81 Mon Sep 17 00:00:00 2001 From: Garrett Rodriguez Date: Mon, 16 Sep 2019 17:54:01 -0700 Subject: [PATCH 02/39] LB-215 Want linux-pkg-build to use package mirror --- branch.config | 14 ++++++++++++++ buildlist.sh | 5 ----- lib/common.sh | 24 ++++++++++++++++++++++++ setup.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 branch.config diff --git a/branch.config b/branch.config new file mode 100644 index 0000000..506ef72 --- /dev/null +++ b/branch.config @@ -0,0 +1,14 @@ +# +# Copyright (c) 2019 by Delphix. All rights reserved. +# + +# +# The "BRANCH" parameter tracks the upstream branch of linux-pkg. It is +# used to determine which branch of the linux package mirror will be used for +# the build if DEFAULT_GIT_BRANCH is not set. DEFAULT_GIT_BRANCH is set when +# linux pkg is built by the linux-pkg-build Jenkins jobs. The +# DEFAULT_GIT_BRANCH parameter should be updated by the release lead on +# branching. +# + +DEFAULT_GIT_BRANCH="master" diff --git a/buildlist.sh b/buildlist.sh index 7e00403..e0b627b 100755 --- a/buildlist.sh +++ b/buildlist.sh @@ -52,11 +52,6 @@ logmust mkdir artifacts # default used if the revision is not set explicitly anywhere else. # export DEFAULT_REVISION="${DEFAULT_REVISION:-$(default_revision)}" -# -# Default branch to checkout when fetching source code for packages. Note that -# this can be overriden by per-package settings. -# -export DEFAULT_GIT_BRANCH="${DEFAULT_GIT_BRANCH:-master}" # # A list of target platform or versions to build modules for can be passed in diff --git a/lib/common.sh b/lib/common.sh index 6a8d702..f32d0b9 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -22,6 +22,30 @@ export DEBIAN_FRONTEND=noninteractive # TODO: allow updating upstream for other branches than master export REPO_UPSTREAM_BRANCH="upstreams/master" +# +# Determine DEFAULT_GIT_BRANCH. If it is unset, default to the branch set in +# branch.config. +# +if [[ -z "$DEFAULT_GIT_BRANCH" ]]; then + echo "DEFAULT_GIT_BRANCH is not set." + if ! source "$TOP/branch.config" 2>/dev/null; then + echo "No branch.config file found in repo root." + exit 1 + fi + + if [[ -z "$DEFAULT_GIT_BRANCH" ]]; then + echo "$DEFAULT_GIT_BRANCH parameter was not sourced from " \ + "branch.config. Ensure branch.config is properly formatted with " \ + "e.g. DEFAULT_GIT_BRANCH=\"\"" + exit 1 + fi + + echo "Defaulting DEFAULT_GIT_BRANCH to branch $DEFAULT_GIT_BRANCH set in" \ + "branch.config." + + export DEFAULT_GIT_BRANCH +fi + # shellcheck disable=SC2086 function enable_colors() { [[ -t 1 ]] && flags="" || flags="-T xterm" diff --git a/setup.sh b/setup.sh index 32339bf..7300069 100755 --- a/setup.sh +++ b/setup.sh @@ -18,7 +18,48 @@ TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" source "$TOP/lib/common.sh" +# +# Update the sources.list file to point to our internal package mirror. If no +# mirror url is passed in, then the latest mirror snapshot is used. +# +configure_apt_sources() { + local package_mirror_url='' + if [[ -n "$DELPHIX_PACKAGE_MIRROR_MAIN" ]]; then + package_mirror_url="$DELPHIX_PACKAGE_MIRROR_MAIN" + else + local latest_url="http://linux-package-mirror.delphix.com/" + latest_url+="${DEFAULT_GIT_BRANCH}/latest/" + package_mirror_url=$(curl -LfSs -o /dev/null -w '%{url_effective}' \ + "$latest_url" || die "Could not curl $latest_url") + + package_mirror_url+="ubuntu" + fi + + # + # Remove other sources in sources.list.d if they are present. + # + [[ -d /etc/apt/sources.list.d ]] && ( + logmust sudo rm -rf /etc/apt/sources.list.d || + die "Could not remove /etc/apt/sources.list.d" + ) + + sudo bash -c "cat <<-EOF >/etc/apt/sources.list +deb ${package_mirror_url} bionic main restricted universe multiverse +deb-src ${package_mirror_url} bionic main restricted universe multiverse + +deb ${package_mirror_url} bionic-updates main restricted universe multiverse +deb-src ${package_mirror_url} bionic-updates main restricted universe multiverse + +deb ${package_mirror_url} bionic-security main restricted universe multiverse +deb-src ${package_mirror_url} bionic-security main restricted universe multiverse + +deb ${package_mirror_url} bionic-backports main restricted universe multiverse +deb-src ${package_mirror_url} bionic-backports main restricted universe multiverse +EOF" || die "/etc/apt/sources.list could not be updated" +} + logmust check_running_system +logmust configure_apt_sources logmust sudo apt-get update # From 48c9ef353cf1e8c7712edcb52397abdd469424f2 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Thu, 3 Oct 2019 13:00:00 -0700 Subject: [PATCH 03/39] Output debug symbol (.ddeb) packages linux-pkg already creates .ddeb packages where appropriate, we just need to copy them to the artifacts/ directory so that they get uploaded with the rest of the packages by Jenkins at the end of the build. --- README.md | 6 +++--- build-info-pkg/build-package.sh | 2 +- lib/common.sh | 2 +- packages/adoptopenjdk/config.sh | 4 ++-- packages/challenge-response/config.sh | 2 +- packages/connstat/config.sh | 2 +- packages/crypt-blowfish/config.sh | 2 +- packages/delphix-platform/config.sh | 2 +- packages/delphix-sso-app/config.sh | 2 +- packages/java8/config.sh | 2 +- packages/td-agent/config.sh | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 6cf4c11..3ab1851 100644 --- a/README.md +++ b/README.md @@ -785,9 +785,9 @@ More instructions available ## Versions and Branches The framework is designed in a way to allow easy integration with the Delphix -release process. The idea is that both the package build artifacts (`.debs`) -and package source code should be available for each Delphix release. This -should hold for both in-house and third-party packages. +release process. The idea is that both the package build artifacts (`.deb`s +and `.ddeb`s) and package source code should be available for each Delphix +release. This should hold for both in-house and third-party packages. Regarding the build artifacts, those should be taken care of by the existing Delphix build artifacts storage policy, available diff --git a/build-info-pkg/build-package.sh b/build-info-pkg/build-package.sh index 40042f0..2ce114f 100755 --- a/build-info-pkg/build-package.sh +++ b/build-info-pkg/build-package.sh @@ -84,6 +84,6 @@ export PACKAGE_REVISION=${DEFAULT_REVISION:-0} logmust set_changelog "$package_name" logmust dpkg-buildpackage -uc -us -b logmust mkdir -p artifacts -logmust mv ../"$package_name"*.deb artifacts/ +logmust mv ../"$package_name"*deb artifacts/ logmust rm -f ../"$package_name"*.buildinfo ../"$package_name"*.changes logmust cp "$INFO_FILE" artifacts/build-info diff --git a/lib/common.sh b/lib/common.sh index f32d0b9..54ce684 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -672,7 +672,7 @@ function dpkg_buildpackage_default() { logmust set_changelog logmust dpkg-buildpackage -b -us -uc logmust cd "$WORKDIR/" - logmust mv ./*.deb artifacts/ + logmust mv ./*deb artifacts/ } # diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh index 77d043b..aae84c1 100644 --- a/packages/adoptopenjdk/config.sh +++ b/packages/adoptopenjdk/config.sh @@ -21,7 +21,7 @@ tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u212b04.tar.gz" jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" function prepare() { - if ! ls "$TOP/packages/make-jpkg/tmp/artifacts/"*.deb >/dev/null 2>&1; then + if ! ls "$TOP/packages/make-jpkg/tmp/artifacts/"*deb >/dev/null 2>&1; then echo_bold "custom java-package not installed. Building package 'make-jpkg' first." logmust "$TOP/buildpkg.sh" make-jpkg fi @@ -40,7 +40,7 @@ function build() { logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" << Date: Mon, 7 Oct 2019 13:54:46 -0700 Subject: [PATCH 04/39] drgn: install build dependencies using info from control file --- packages/drgn/config.sh | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index e66a052..c5b94f6 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -23,24 +23,17 @@ UPSTREAM_GIT_URL="https://github.com/osandov/drgn.git" UPSTREAM_GIT_BRANCH="master" function prepare() { - logmust install_pkgs \ - autoconf \ - automake \ - bison \ - flex \ - gawk \ - git \ - gcc \ - libbz2-dev \ - libdw-dev \ - libelf-dev \ - liblzma-dev \ - libtool \ - make \ - pkg-config \ - python3-distutils \ - python3.6-dev \ - zlib1g-dev + # + # Strictly speaking libkdumpfile is not a hard prerequisite for + # drgn itself, but it is a hard requirement in our use-case as + # we do want to use drgn for kdump-compressed crash dumps. + # + if ! dpkg-query --show libkdumpfile >/dev/null 2>&1; then + echo_bold "libkdumpfile not installed. Building package 'libkdumpfile' first." + logmust "$TOP/buildpkg.sh" libkdumpfile + fi + + logmust install_build_deps_from_control_file } function build() { From 7ab528eac46badf13e40940c50854fb445ffa654 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Mon, 18 Nov 2019 09:32:18 -0800 Subject: [PATCH 05/39] Add performance-diagnostics package --- package-lists/build/userland.pkgs | 1 + packages/performance-diagnostics/config.sh | 29 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 packages/performance-diagnostics/config.sh diff --git a/package-lists/build/userland.pkgs b/package-lists/build/userland.pkgs index c9e89a7..34fcec5 100644 --- a/package-lists/build/userland.pkgs +++ b/package-lists/build/userland.pkgs @@ -26,5 +26,6 @@ nfs-utils python-rtslib-fb sdb targetcli-fb +performance-diagnostics ptools td-agent-prebuilt diff --git a/packages/performance-diagnostics/config.sh b/packages/performance-diagnostics/config.sh new file mode 100644 index 0000000..0d6e978 --- /dev/null +++ b/packages/performance-diagnostics/config.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2019 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 +# + +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/performance-diagnostics.git" +DEFAULT_PACKAGE_VERSION="1.0.0" + +function prepare() { + logmust install_build_deps_from_control_file +} + +function build() { + logmust dpkg_buildpackage_default +} From de5ee9f67145010b9548ff4bc90c2b4399a1b5c1 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Mon, 18 Nov 2019 16:56:46 -0500 Subject: [PATCH 06/39] DLPX-67499 Use the HWE kernel for the generic platform on Ubuntu 18.04 --- README.md | 3 ++- lib/common.sh | 26 ++++++++++++++++++++------ setup.sh | 16 ++++++++-------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 3ab1851..2175781 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,8 @@ There's a set of environment variables that can be set to modify the operation of some of the scripts defined above. * **DISABLE_SYSTEM_CHECK**: Set to "true" to disable the check that makes sure - we are running on an Ubuntu Bionic (18.04) system in AWS. Affects all scripts. + we are running on the appropriate Ubuntu distribution in AWS. + Affects all scripts. * **DRY_RUN**: Set to "true" to prevent `updatelist.sh` from updating production package repositories. `updatelist.sh` will invoke `push-updates.sh` with `-n`. diff --git a/lib/common.sh b/lib/common.sh index 54ce684..6ece944 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -22,6 +22,8 @@ export DEBIAN_FRONTEND=noninteractive # TODO: allow updating upstream for other branches than master export REPO_UPSTREAM_BRANCH="upstreams/master" +export UBUNTU_DISTRIBUTION="bionic" + # # Determine DEFAULT_GIT_BRANCH. If it is unset, default to the branch set in # branch.config. @@ -103,9 +105,9 @@ function logmust() { } # -# Check that we are running on an Ubuntu Bionic system in AWS. -# This is not a strict requirement for the build to work but rather a -# safety measure to prevent developers from accidentally running the +# Check that we are running in AWS on an Ubuntu system of the appropriate +# distribution. This is not a strict requirement for the build to work but +# rather a safety measure to prevent developers from accidentally running the # scripts on their work system and changing its configuration. # function check_running_system() { @@ -115,8 +117,8 @@ function check_running_system() { fi if ! (command -v lsb_release >/dev/null && - [[ $(lsb_release -cs) == "bionic" ]]); then - die "Script can only be ran on an ubuntu-bionic system." + [[ $(lsb_release -cs) == "$UBUNTU_DISTRIBUTION" ]]); then + die "Script can only be ran on an ubuntu-${UBUNTU_DISTRIBUTION} system." fi if ! curl "http://169.254.169.254/latest/meta-datas" \ @@ -695,6 +697,7 @@ function store_git_info() { # function get_kernel_for_platform() { local platform="$1" + local package # # For each supported platform, Ubuntu provides a 'linux-image-PLATFORM' @@ -705,10 +708,21 @@ function get_kernel_for_platform() { # for kernel version '4.15.0-1027-aws'. We use this depenency to figure # out the default kernel version for a given platform. # + # The "generic" platform is a special case, since we want to use the + # hwe kernel image instead of the regular generic image. + # # Note that while the default kernel is usually also the latest # available, it is not always the case. # - if [[ "$(apt-cache show --no-all-versions "linux-image-${platform}" \ + + if [[ "$platform" == generic ]] && + [[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then + package=linux-image-generic-hwe-18.04 + else + package="linux-image-${platform}" + fi + + if [[ "$(apt-cache show --no-all-versions "$package" \ 2>/dev/null | grep Depends)" =~ linux-image-([^,]*-${platform}) ]]; then _RET=${BASH_REMATCH[1]} return 0 diff --git a/setup.sh b/setup.sh index 7300069..7a98c99 100755 --- a/setup.sh +++ b/setup.sh @@ -44,17 +44,17 @@ configure_apt_sources() { ) sudo bash -c "cat <<-EOF >/etc/apt/sources.list -deb ${package_mirror_url} bionic main restricted universe multiverse -deb-src ${package_mirror_url} bionic main restricted universe multiverse +deb ${package_mirror_url} ${UBUNTU_DISTRIBUTION} main restricted universe multiverse +deb-src ${package_mirror_url} ${UBUNTU_DISTRIBUTION} main restricted universe multiverse -deb ${package_mirror_url} bionic-updates main restricted universe multiverse -deb-src ${package_mirror_url} bionic-updates main restricted universe multiverse +deb ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-updates main restricted universe multiverse +deb-src ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-updates main restricted universe multiverse -deb ${package_mirror_url} bionic-security main restricted universe multiverse -deb-src ${package_mirror_url} bionic-security main restricted universe multiverse +deb ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-security main restricted universe multiverse +deb-src ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-security main restricted universe multiverse -deb ${package_mirror_url} bionic-backports main restricted universe multiverse -deb-src ${package_mirror_url} bionic-backports main restricted universe multiverse +deb ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-backports main restricted universe multiverse +deb-src ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-backports main restricted universe multiverse EOF" || die "/etc/apt/sources.list could not be updated" } From fcb13c94014273ee94596591381702944c9dda3e Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Mon, 9 Dec 2019 09:48:02 -0800 Subject: [PATCH 07/39] add latest crash-util --- package-lists/build/userland.pkgs | 1 + package-lists/update/userland.pkgs | 1 + packages/crash/config.sh | 35 ++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 packages/crash/config.sh diff --git a/package-lists/build/userland.pkgs b/package-lists/build/userland.pkgs index 34fcec5..061e074 100644 --- a/package-lists/build/userland.pkgs +++ b/package-lists/build/userland.pkgs @@ -16,6 +16,7 @@ delphix-sso-app bpftrace challenge-response cloud-init +crash crash-python crypt-blowfish delphix-platform diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs index 243bdcc..4b71186 100644 --- a/package-lists/update/userland.pkgs +++ b/package-lists/update/userland.pkgs @@ -13,6 +13,7 @@ bpftrace cloud-init +crash drgn nfs-utils python-rtslib-fb diff --git a/packages/crash/config.sh b/packages/crash/config.sh new file mode 100644 index 0000000..5f709a5 --- /dev/null +++ b/packages/crash/config.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright 2019 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/crash.git" +DEFAULT_PACKAGE_VERSION=1.0.0 + +UPSTREAM_GIT_URL="https://github.com/crash-utility/crash.git" +UPSTREAM_GIT_BRANCH="master" + +function prepare() { + logmust install_build_deps_from_control_file +} + +function build() { + logmust dpkg_buildpackage_default +} + +function update_upstream() { + logmust update_upstream_from_git +} From e9e5a3802342f83fd8f4b34f1c52506daa80c471 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 27 Dec 2019 13:56:56 -0500 Subject: [PATCH 08/39] temporarily disable auto-update of crash package --- package-lists/update/userland.pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs index 4b71186..8dfad1a 100644 --- a/package-lists/update/userland.pkgs +++ b/package-lists/update/userland.pkgs @@ -13,7 +13,7 @@ bpftrace cloud-init -crash +#crash drgn nfs-utils python-rtslib-fb From 07aef3aedba2fcd87f84d0b39ea18440778ade61 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 3 Jan 2020 15:00:57 -0500 Subject: [PATCH 09/39] Revert "temporarily disable auto-update of crash package" This reverts commit e9e5a3802342f83fd8f4b34f1c52506daa80c471. --- package-lists/update/userland.pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs index 8dfad1a..4b71186 100644 --- a/package-lists/update/userland.pkgs +++ b/package-lists/update/userland.pkgs @@ -13,7 +13,7 @@ bpftrace cloud-init -#crash +crash drgn nfs-utils python-rtslib-fb From d6b8db33d036db651ea49b75dc32cc9a2c6ff368 Mon Sep 17 00:00:00 2001 From: George Wilson Date: Mon, 6 Jan 2020 15:03:23 -0500 Subject: [PATCH 10/39] DLPX-67915 Add oracle-specific kernel --- lib/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common.sh b/lib/common.sh index 6ece944..3f35f52 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018, 2019 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -750,7 +750,7 @@ function determine_target_kernels() { return 0 fi - local supported_platforms="generic aws gcp azure kvm" + local supported_platforms="generic aws gcp azure oracle" local platform if [[ -z "$TARGET_PLATFORMS" ]]; then From 05e0d7a7e232ac17b2aa88086ab97e6dd948002b Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 12 Feb 2020 10:24:41 -0500 Subject: [PATCH 11/39] DLPX-68435 crash dump fails on 5.3 kernel Enable auto-update for the makedumpfile package and do not remove kdump-tools to keep both packages built from the same source. --- package-lists/update/userland.pkgs | 1 + packages/makedumpfile/config.sh | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs index 4b71186..56968c5 100644 --- a/package-lists/update/userland.pkgs +++ b/package-lists/update/userland.pkgs @@ -15,6 +15,7 @@ bpftrace cloud-init crash drgn +makedumpfile nfs-utils python-rtslib-fb targetcli-fb diff --git a/packages/makedumpfile/config.sh b/packages/makedumpfile/config.sh index d8b5343..03676b3 100644 --- a/packages/makedumpfile/config.sh +++ b/packages/makedumpfile/config.sh @@ -28,11 +28,9 @@ function prepare() { function build() { logmust cd "$WORKDIR/repo" if [[ -z "$PACKAGE_VERSION" ]]; then - logmust eval PACKAGE_VERSION="$(grep '^VERSION=' Makefile | cut -d "=" -f 2)" + logmust eval PACKAGE_VERSION="1:$(grep '^VERSION=' Makefile | cut -d "=" -f 2)" fi logmust dpkg_buildpackage_default - # we only need makedumpfile package - logmust rm -f "$WORKDIR/artifacts/"kdump-tools_*_amd64.deb } function update_upstream() { From c5af854d7ca83fb5a06761e127d9f61b61eb1bcd Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 12 Feb 2020 15:05:14 -0500 Subject: [PATCH 12/39] Deprecating DEFAULT_PACKAGE_GIT_BRANCH Given our versioning model, where each package managed by linux-pkg is tagged and branched according to releases, we should rely on the DEFAULT_BRANCH parameter for fetching the source code for each package. Setting DEFAULT_PACKAGE_GIT_BRANCH is dangerous for 2 reasons: - First because a release tag for that repository then becomes misleading, as it is not what is used to rebuild that repository for a particular release version. - Second, if changes are later pushed to that branch, then rebuilding the original release will pull those new changes. For now, DEFAULT_PACKAGE_GIT_BRANCH is being deprecated rather than removed. We will evaluate whether it still has value for testing purposes or for projects such as SUV. --- README.md | 5 ++++- packages/crash-python/config.sh | 1 - packages/gdb-python/config.sh | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2175781..dc63462 100644 --- a/README.md +++ b/README.md @@ -344,11 +344,14 @@ Here is a list of variables that can be defined for a package: `https://` URL. One exception is if the source of the package being built isn't fetched from git. In this case, set this to "none". -* **DEFAULT_PACKAGE_GIT_BRANCH**: (Optional) Default git branch to use when +* **DEFAULT_PACKAGE_GIT_BRANCH**: (DEPRECATED) Default git branch to use when fetching from or pushing to `DEFAULT_PACKAGE_GIT_URL`. This should be typically left unset. The branch to fetch the package from defaults to the value of the environment variable `DEFAULT_BRANCH`, which itself defaults to "master". + WARNING: do not set this parameter unless you know exactly what you are doing, + as our current versioning convention is to use DEFAULT_BRANCH for each + package. This parameter may be removed in the future. * **DEFAULT_PACKAGE_VERSION**: (Optional) The version of the package is set to this value when it is built. **Note:** If this field is not set, then you diff --git a/packages/crash-python/config.sh b/packages/crash-python/config.sh index 6183fee..9449bb2 100644 --- a/packages/crash-python/config.sh +++ b/packages/crash-python/config.sh @@ -17,7 +17,6 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/os-platform/crash-python.git" -DEFAULT_PACKAGE_GIT_BRANCH="next" DEFAULT_PACKAGE_VERSION=1.0.0 function prepare() { diff --git a/packages/gdb-python/config.sh b/packages/gdb-python/config.sh index 5eaa86f..aea4a7c 100644 --- a/packages/gdb-python/config.sh +++ b/packages/gdb-python/config.sh @@ -17,7 +17,6 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/os-platform/gdb-python.git" -DEFAULT_PACKAGE_GIT_BRANCH="master-suse-target" DEFAULT_PACKAGE_VERSION=1.0.0 function prepare() { From 7a72caec67aaa06d2e0f52f83724d50892cfafb9 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Mon, 2 Mar 2020 11:28:37 -0800 Subject: [PATCH 13/39] libkdumpfile: abstract out dependencies and sync with upstream --- package-lists/update/userland.pkgs | 1 + packages/libkdumpfile/config.sh | 21 +++++++++------------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs index 56968c5..c4c582e 100644 --- a/package-lists/update/userland.pkgs +++ b/package-lists/update/userland.pkgs @@ -15,6 +15,7 @@ bpftrace cloud-init crash drgn +libkdumpfile makedumpfile nfs-utils python-rtslib-fb diff --git a/packages/libkdumpfile/config.sh b/packages/libkdumpfile/config.sh index 240fa40..20aa930 100644 --- a/packages/libkdumpfile/config.sh +++ b/packages/libkdumpfile/config.sh @@ -16,21 +16,14 @@ # # shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/os-platform/libkdumpfile.git" +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/libkdumpfile.git" DEFAULT_PACKAGE_VERSION=1.0.0 +UPSTREAM_GIT_URL="https://github.com/ptesarik/libkdumpfile.git" +UPSTREAM_GIT_BRANCH="master" + function prepare() { - logmust install_pkgs \ - autoconf \ - automake \ - git \ - liblzo2-dev \ - libsnappy1v5 \ - libtool \ - pkg-config \ - python3-distutils \ - python3.6-dev \ - zlib1g-dev + logmust install_build_deps_from_control_file } function build() { @@ -39,3 +32,7 @@ function build() { # Install libkdumpfile, it's needed to build drgn logmust install_pkgs "$WORKDIR/artifacts"/*.deb } + +function update_upstream() { + logmust update_upstream_from_git +} From 0fa2b719ebb22ad0f90568407ec1776173503c56 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Mon, 13 Apr 2020 17:07:56 -0400 Subject: [PATCH 14/39] LB-256 Use github actions to perform sanity checks on pull-requests for linux-pkg --- .github/scripts/install-shellcheck.sh | 4 ++++ .github/scripts/install-shfmt.sh | 5 +++++ .github/workflows/main.yml | 15 +++++++++++++++ 3 files changed, 24 insertions(+) create mode 100755 .github/scripts/install-shellcheck.sh create mode 100755 .github/scripts/install-shfmt.sh create mode 100644 .github/workflows/main.yml diff --git a/.github/scripts/install-shellcheck.sh b/.github/scripts/install-shellcheck.sh new file mode 100755 index 0000000..15348ec --- /dev/null +++ b/.github/scripts/install-shellcheck.sh @@ -0,0 +1,4 @@ +#!/bin/bash -ex + +apt-get update +apt-get install -y shellcheck diff --git a/.github/scripts/install-shfmt.sh b/.github/scripts/install-shfmt.sh new file mode 100755 index 0000000..f756709 --- /dev/null +++ b/.github/scripts/install-shfmt.sh @@ -0,0 +1,5 @@ +#!/bin/bash -ex + +wget -nv -O /usr/local/bin/shfmt \ + https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_amd64 +chmod +x /usr/local/bin/shfmt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..8847eb3 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,15 @@ +on: [push, pull_request] + +jobs: + check-shellcheck: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: sudo ./.github/scripts/install-shellcheck.sh + - run: make shellcheck + check-shfmt: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: sudo ./.github/scripts/install-shfmt.sh + - run: make shfmtcheck From 7ce81cc611e6e402b19d713b4c4bc4cb321a3295 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Mon, 27 Apr 2020 13:10:11 -0700 Subject: [PATCH 15/39] Add grub2 package to the Delphix build process (#85) --- package-lists/build/kernel.pkgs | 7 +++- package-lists/update/userland.pkgs | 1 + packages/grub2/config.sh | 55 ++++++++++++++++++++++++++++++ packages/zfs/config.sh | 3 ++ 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 packages/grub2/config.sh diff --git a/package-lists/build/kernel.pkgs b/package-lists/build/kernel.pkgs index 2e66a10..a0ca034 100644 --- a/package-lists/build/kernel.pkgs +++ b/package-lists/build/kernel.pkgs @@ -3,6 +3,11 @@ # kernel versions. # +# Note: The following packages should be built first because other packages +# depend on them being built. +# - zfs is required by grub2 +zfs + connstat delphix-kernel -zfs +grub2 diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs index c4c582e..f0bcbaa 100644 --- a/package-lists/update/userland.pkgs +++ b/package-lists/update/userland.pkgs @@ -21,3 +21,4 @@ nfs-utils python-rtslib-fb targetcli-fb td-agent +grub2 diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh new file mode 100644 index 0000000..1603a5f --- /dev/null +++ b/packages/grub2/config.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/grub2" + +UPSTREAM_GIT_URL=https://git.launchpad.net/ubuntu/+source/grub2 +UPSTREAM_GIT_BRANCH=applied/ubuntu/bionic-updates + +# +# Install build dependencies for the package. +# +function prepare() { + if ! dpkg-query --show libzfslinux-dev >/dev/null 2>&1; then + echo_bold "libzfs not installed. Building package 'zfs' first." + logmust "$TOP/buildpkg.sh" zfs + fi + + logmust install_build_deps_from_control_file + return +} + +# +# Build the package. +# +function build() { + logmust cd "$WORKDIR/repo" + if [[ -z "$PACKAGE_VERSION" ]]; then + logmust eval PACKAGE_VERSION="$(dpkg-parsechangelog -S Version | \ + awk -F'-' '{print $1}')" + fi + logmust dpkg_buildpackage_default +} + +# +# Hook to fetch upstream package changes and merge into our tree. +# +function update_upstream() { + logmust update_upstream_from_git + return +} diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index 7bc37ea..2d02fc2 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -162,6 +162,9 @@ function build() { done logmust cd "$WORKDIR" logmust mv "all-packages/"*.deb "artifacts/" + + # Install libzfs which is required to build grub + logmust install_pkgs "$WORKDIR/artifacts"/{libnvpair1linux,libuutil1linux,libzfs2linux,libzpool2linux,libzfslinux-dev}_*.deb } function update_upstream() { From 2b48c37404708a21d9bca27aa1ae8a59790a9d51 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Mon, 27 Apr 2020 17:09:59 -0400 Subject: [PATCH 16/39] LB-290 Add query-packages script to linux-pkg (#89) --- .github/scripts/verify-query-packages.sh | 63 ++++++++++ .github/workflows/main.yml | 16 +++ buildpkg.sh | 2 +- lib/common.sh | 69 +++++++---- query-packages.sh | 151 +++++++++++++++++++++++ setup.sh | 2 + 6 files changed, 276 insertions(+), 27 deletions(-) create mode 100755 .github/scripts/verify-query-packages.sh create mode 100755 query-packages.sh diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh new file mode 100755 index 0000000..c795a16 --- /dev/null +++ b/.github/scripts/verify-query-packages.sh @@ -0,0 +1,63 @@ +#!/bin/bash -ex +# shellcheck disable=SC2012 + +set -o pipefail + +cd "$(git rev-parse --show-toplevel)" + +# Make sure a basic command doesn't print anything to stderr +test -z "$(./query-packages.sh single -o name,git-url zfs 2>&1 >/dev/null)" + +# Expect: "zfs https://github.com/delphix/zfs.git" +read -r -a fields <<<"$(./query-packages.sh single -o name,git-url zfs 2>&1)" +test ${#fields[@]} -eq 2 +test "${fields[0]}" == 'zfs' +test "${fields[1]}" == 'https://github.com/delphix/zfs.git' + +# Expect: "https://github.com/delphix/zfs.git zfs" +read -r -a fields <<<"$(./query-packages.sh single -o git-url,name zfs 2>&1)" +test ${#fields[@]} -eq 2 +test "${fields[0]}" == 'https://github.com/delphix/zfs.git' +test "${fields[1]}" == 'zfs' + +# Expect: "zfs" +read -r -a fields <<<"$(./query-packages.sh single zfs 2>&1)" +test ${#fields[@]} -eq 1 +test "${fields[0]}" == 'zfs' + +# Expect: "https://github.com/delphix/zfs.git" +read -r -a fields <<<"$(./query-packages.sh single -o git-url zfs 2>&1)" +test ${#fields[@]} -eq 1 +test "${fields[0]}" == 'https://github.com/delphix/zfs.git' + +# Expect that "list all" outputs all directory names under packages/ +diff <(ls -1 packages | sort) <(./query-packages.sh list all 2>&1 | sort) + +# Expect that outputing git-url for all packages works and that the output +# length corresponds to the number of packages. +test "$(ls -1 packages | wc -l)" -eq "$(./query-packages.sh list -o name,git-url all 2>&1 | wc -l)" + +# Check that all package lists under package-lists\ can be loaded and that each +# line of the output of the command actually refers to a package. +find package-lists -name '*.pkgs' | while read -r list; do + list="${list#package-lists/}" + ./query-packages.sh list "$list" 2>&1 | ( + cd packages + xargs ls + ) >/dev/null +done + +# Check that querying "appliance" list works +./query-packages.sh list appliance >/dev/null + +# Check that the output from the appliance list contains zfs and +# delphix-platform packages. Note, we explicitly do not use grep -q here as it +# exits as soon as a match is found and that causes a broken pipe error as +# query-packages attempts to write more output. +./query-packages.sh list appliance | grep zfs >/dev/null +./query-packages.sh list appliance | grep delphix-platform >/dev/null + +# Check that executing query-packages works from another directory. +# This redoes the "list all" test from above +cd packages +diff <(ls -1 | sort) <(../query-packages.sh list all 2>&1 | sort) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8847eb3..978100f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,3 +13,19 @@ jobs: - uses: actions/checkout@v1 - run: sudo ./.github/scripts/install-shfmt.sh - run: make shfmtcheck + verify-query-packages: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: ./.github/scripts/verify-query-packages.sh + verify-query-packages-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v1 + - run: ./.github/scripts/verify-query-packages.sh + verify-query-packages-jenkins: + runs-on: ubuntu-latest + container: jenkins/jenkins + steps: + - uses: actions/checkout@v1 + - run: ./.github/scripts/verify-query-packages.sh diff --git a/buildpkg.sh b/buildpkg.sh index 06d49e6..76d70d4 100755 --- a/buildpkg.sh +++ b/buildpkg.sh @@ -178,7 +178,7 @@ logmust check_package_exists "$PACKAGE" # If the script is called manually, we set it here. # DEFAULT_REVISION="${DEFAULT_REVISION:-$(default_revision)}" -DEFAULT_GIT_BRANCH="${DEFAULT_GIT_BRANCH:-master}" +logmust determine_default_git_branch echo "" echo_bold "====================================================================" diff --git a/lib/common.sh b/lib/common.sh index 3f35f52..325ba38 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -24,30 +24,6 @@ export REPO_UPSTREAM_BRANCH="upstreams/master" export UBUNTU_DISTRIBUTION="bionic" -# -# Determine DEFAULT_GIT_BRANCH. If it is unset, default to the branch set in -# branch.config. -# -if [[ -z "$DEFAULT_GIT_BRANCH" ]]; then - echo "DEFAULT_GIT_BRANCH is not set." - if ! source "$TOP/branch.config" 2>/dev/null; then - echo "No branch.config file found in repo root." - exit 1 - fi - - if [[ -z "$DEFAULT_GIT_BRANCH" ]]; then - echo "$DEFAULT_GIT_BRANCH parameter was not sourced from " \ - "branch.config. Ensure branch.config is properly formatted with " \ - "e.g. DEFAULT_GIT_BRANCH=\"\"" - exit 1 - fi - - echo "Defaulting DEFAULT_GIT_BRANCH to branch $DEFAULT_GIT_BRANCH set in" \ - "branch.config." - - export DEFAULT_GIT_BRANCH -fi - # shellcheck disable=SC2086 function enable_colors() { [[ -t 1 ]] && flags="" || flags="-T xterm" @@ -83,7 +59,7 @@ function without_colors() { } function echo_error() { - echo -e "${FMT_BOLD}${FMT_RED}Error: $*${FMT_NF}" + echo -e "${FMT_BOLD}${FMT_RED}Error: $*${FMT_NF}" >&2 } function echo_success() { @@ -128,6 +104,32 @@ function check_running_system() { fi } +# +# Determine DEFAULT_GIT_BRANCH. If it is unset, default to the branch set in +# branch.config. +# +function determine_default_git_branch() { + + [[ -n "$DEFAULT_GIT_BRANCH" ]] && return + + echo "DEFAULT_GIT_BRANCH is not set." + if ! source "$TOP/branch.config" 2>/dev/null; then + die "No branch.config file found in repo root." + fi + + if [[ -z "$DEFAULT_GIT_BRANCH" ]]; then + die "$DEFAULT_GIT_BRANCH parameter was not sourced" \ + "from branch.config. Ensure branch.config is" \ + "properly formatted with e.g." \ + "DEFAULT_GIT_BRANCH=''" + fi + + echo "Defaulting DEFAULT_GIT_BRANCH to branch" \ + "$DEFAULT_GIT_BRANCH set in branch.config." + + export DEFAULT_GIT_BRANCH +} + function check_package_exists() { local pkg="$1" @@ -443,6 +445,19 @@ function install_build_deps_from_control_file() { logmust popd } +function list_all_packages() { + local pkg + + _RET_LIST=() + + for pkg in "$TOP/packages/"*; do + pkg=$(basename "$pkg") + if [[ -f "$TOP/packages/$pkg/config.sh" ]]; then + _RET_LIST+=("$pkg") + fi + done +} + function read_package_list() { local file="$1" @@ -451,9 +466,11 @@ function read_package_list() { _RET_LIST=() + [[ -f "$file" ]] || die "Not a file: $file" + while read -r line; do # trim whitespace - pkg=$(echo "$line" | sed 's/^\s*//;s/\s*$//') + pkg=$(echo "$line" | tr -d '[:space:]') [[ -z "$pkg" ]] && continue # ignore comments [[ ${pkg:0:1} == "#" ]] && continue diff --git a/query-packages.sh b/query-packages.sh new file mode 100755 index 0000000..14fe386 --- /dev/null +++ b/query-packages.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 +TOP="$PWD" +source "$TOP/lib/common.sh" + +ALL_OUTPUT_FIELDS=(name git-url dependencies) + +function usage() { + local output_fields="${ALL_OUTPUT_FIELDS[*]}" + ( + [[ $# != 0 ]] && ( + echo "$(basename "$0"): $*" + echo + ) + echo "Usage:" + echo " $(basename "$0") single [-o FIELDS] PACKAGE" + echo " $(basename "$0") list [-o FIELDS] LIST_FILE" + echo "" + echo " Either display information about a single package" + echo " or a package list located under the package-lists" + echo " directory. You can also pass one of the following" + echo " special values to the list command:" + echo " all: displays all known packages" + echo " appliance: displays all packages used by appliance" + echo "" + echo " -o Comma delimited output fields." + echo " Possible values: ${output_fields// /, }." + echo " By default, only print package name." + echo " -h Show this help." + ) >&2 + exit 2 +} + +function print_package() { + local pkgname="$1" + local outarray=() + ( + local field + load_package_config "$pkgname" >/dev/null + for field in "${ACTIVE_OUTPUT_FIELDS[@]}"; do + case "$field" in + name) outarray+=("$pkgname") ;; + git-url) outarray+=("${DEFAULT_PACKAGE_GIT_URL:-none}") ;; + dependencies) outarray+=(none) ;; + esac + done + IFS=$'\t' + echo "${outarray[*]}" + ) +} + +function query_list() { + local list="$1" + local dups + + # Package list is returned in _RET_LIST by the functions below. + # Note that we should always redirect stdout to /dev/null when + # calling lib/common.sh functions as they may generate unwanted + # debug output. + if [[ $list == all ]]; then + list_all_packages >/dev/null + elif [[ $list == appliance ]]; then + # concatenate kernel and userland packages + read_package_list "$TOP/package-lists/build/kernel.pkgs" >/dev/null + local kernel_list=("${_RET_LIST[@]}") + read_package_list "$TOP/package-lists/build/userland.pkgs" >/dev/null + _RET_LIST+=("${kernel_list[@]}") + # check that there are no duplicates + dups=$(printf '%s\n' "${_RET_LIST[@]}" | sort | uniq -d) + [[ -z $dups ]] || die "Some apliance packages appear in both" \ + "build/kernel.pkgs and build/userland.pkgs:\\n${dups}" + else + read_package_list "$TOP/package-lists/${list}" >/dev/null + fi +} + +function do_list() { + local list="$1" + local pkgname + + query_list "$list" + for pkgname in "${_RET_LIST[@]}"; do + print_package "$pkgname" + done +} + +function do_single() { + local pkgname="$1" + [[ -n $pkgname ]] || usage "missing argument for pkgname" + + check_package_exists "$pkgname" >/dev/null + print_package "$pkgname" +} + +case "$1" in +list) target=do_list ;; +single) target=do_single ;; +*) usage >&2 ;; +esac +shift + +opt_o="" +while getopts ':ho:' c; do + case "$c" in + o) + [[ -z $opt_o ]] || usage "-o appears more than once" + opt_o="$OPTARG" + ;; + h) usage ;; + *) usage "illegal option -- $OPTARG" ;; + esac +done +shift $((OPTIND - 1)) + +# +# Parse list of fields to output +# +if [[ -n $opt_o ]]; then + ACTIVE_OUTPUT_FIELDS=() + for field in $(echo "$opt_o" | tr ',' ' '); do + for f in "${ALL_OUTPUT_FIELDS[@]}"; do + [[ $f == "$field" ]] && break + done + if [[ $f == "$field" ]]; then + ACTIVE_OUTPUT_FIELDS+=("$field") + else + usage "invalid output field '$field'" + fi + done +else + # By default, only print package name + ACTIVE_OUTPUT_FIELDS=(name) +fi + +$target "$@" diff --git a/setup.sh b/setup.sh index 7a98c99..b300088 100755 --- a/setup.sh +++ b/setup.sh @@ -18,6 +18,8 @@ TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" source "$TOP/lib/common.sh" +logmust determine_default_git_branch + # # Update the sources.list file to point to our internal package mirror. If no # mirror url is passed in, then the latest mirror snapshot is used. From 86f1dfa877c7f0a9c6a3c0eca411ff5aaa1fc858 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Mon, 27 Apr 2020 18:10:25 -0400 Subject: [PATCH 17/39] fix shfmt (#90) --- packages/grub2/config.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh index 1603a5f..0fa91ff 100644 --- a/packages/grub2/config.sh +++ b/packages/grub2/config.sh @@ -39,9 +39,9 @@ function prepare() { # function build() { logmust cd "$WORKDIR/repo" - if [[ -z "$PACKAGE_VERSION" ]]; then - logmust eval PACKAGE_VERSION="$(dpkg-parsechangelog -S Version | \ - awk -F'-' '{print $1}')" + if [[ -z $PACKAGE_VERSION ]]; then + logmust eval PACKAGE_VERSION="$(dpkg-parsechangelog -S Version | + awk -F'-' '{print $1}')" fi logmust dpkg_buildpackage_default } From df39806a95b2906c68dccef5a3018ab6be96b948 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 13 May 2020 11:07:48 -0700 Subject: [PATCH 18/39] delphix-platform: "make check" is no longer supported (#94) --- packages/delphix-platform/config.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/delphix-platform/config.sh b/packages/delphix-platform/config.sh index 1e4baba..69a6cc0 100644 --- a/packages/delphix-platform/config.sh +++ b/packages/delphix-platform/config.sh @@ -19,12 +19,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-platform.git" DEFAULT_PACKAGE_VERSION="1.0.0" -function checkstyle() { - logmust cd "$WORKDIR/repo" - logmust ansible-playbook bootstrap/playbook.yml - logmust ./scripts/docker-run.sh make check -} - function build() { logmust cd "$WORKDIR/repo" logmust ansible-playbook bootstrap/playbook.yml From f98c866d6bcbed0d50e095d01cfb907f67af598b Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Mon, 15 Jun 2020 19:05:23 -0700 Subject: [PATCH 19/39] Recov env package (#92) --- package-lists/build/userland.pkgs | 1 + packages/recovery-environment/config.sh | 29 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 packages/recovery-environment/config.sh diff --git a/package-lists/build/userland.pkgs b/package-lists/build/userland.pkgs index 061e074..5f3e1fd 100644 --- a/package-lists/build/userland.pkgs +++ b/package-lists/build/userland.pkgs @@ -30,3 +30,4 @@ targetcli-fb performance-diagnostics ptools td-agent-prebuilt +recovery-environment diff --git a/packages/recovery-environment/config.sh b/packages/recovery-environment/config.sh new file mode 100644 index 0000000..5f617ed --- /dev/null +++ b/packages/recovery-environment/config.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/recovery-environment.git" +DEFAULT_PACKAGE_VERSION=1.0.0 + +function prepare() { + logmust install_build_deps_from_control_file + return +} + +function build() { + logmust dpkg_buildpackage_default +} From 34f4250a189a3fd042293ed2e9b2ef90aae22c89 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Thu, 18 Jun 2020 12:02:35 -0700 Subject: [PATCH 20/39] Upstream libkdumpfile branch name update (#98) Recently libkdumpfile changed its main branch name from `master` to `tip`. Update the branch definition in our repo to reflect this change. --- packages/libkdumpfile/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/libkdumpfile/config.sh b/packages/libkdumpfile/config.sh index 20aa930..7710e2d 100644 --- a/packages/libkdumpfile/config.sh +++ b/packages/libkdumpfile/config.sh @@ -20,7 +20,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/libkdumpfile.git" DEFAULT_PACKAGE_VERSION=1.0.0 UPSTREAM_GIT_URL="https://github.com/ptesarik/libkdumpfile.git" -UPSTREAM_GIT_BRANCH="master" +UPSTREAM_GIT_BRANCH="tip" function prepare() { logmust install_build_deps_from_control_file From e8c438e52a6a74c80e6de4e933f35a50059ee697 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Wed, 1 Jul 2020 15:25:28 -0700 Subject: [PATCH 21/39] route zfs package into recovery environment (#102) --- package-lists/build/kernel.pkgs | 3 ++- package-lists/build/userland.pkgs | 1 - packages/recovery-environment/config.sh | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/package-lists/build/kernel.pkgs b/package-lists/build/kernel.pkgs index a0ca034..50be040 100644 --- a/package-lists/build/kernel.pkgs +++ b/package-lists/build/kernel.pkgs @@ -5,9 +5,10 @@ # Note: The following packages should be built first because other packages # depend on them being built. -# - zfs is required by grub2 +# - zfs is required by grub2 and recovery-environment zfs connstat delphix-kernel grub2 +recovery-environment diff --git a/package-lists/build/userland.pkgs b/package-lists/build/userland.pkgs index 5f3e1fd..061e074 100644 --- a/package-lists/build/userland.pkgs +++ b/package-lists/build/userland.pkgs @@ -30,4 +30,3 @@ targetcli-fb performance-diagnostics ptools td-agent-prebuilt -recovery-environment diff --git a/packages/recovery-environment/config.sh b/packages/recovery-environment/config.sh index 5f617ed..b5b0bf5 100644 --- a/packages/recovery-environment/config.sh +++ b/packages/recovery-environment/config.sh @@ -19,8 +19,15 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/recovery-environment.git" DEFAULT_PACKAGE_VERSION=1.0.0 +ZFS_DEB_PATH="$TOP/packages/zfs/tmp/artifacts" function prepare() { + if [ ! "$(ls -A "$ZFS_DEB_PATH")" ]; then + logmust "$TOP/buildpkg.sh" zfs + fi + logmust install_build_deps_from_control_file + logmust mkdir "$WORKDIR/repo/external-debs/" + logmust cp "$ZFS_DEB_PATH/"*.deb "$WORKDIR/repo/external-debs/" return } From 0297ae5ec949851b51d2b675ac7415257a7efd7a Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 9 Jul 2020 08:54:26 -0700 Subject: [PATCH 22/39] Stop updating "td-agent" due to build failure (#104) Closes #103 --- package-lists/update/userland.pkgs | 1 - 1 file changed, 1 deletion(-) diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs index f0bcbaa..c9b588c 100644 --- a/package-lists/update/userland.pkgs +++ b/package-lists/update/userland.pkgs @@ -20,5 +20,4 @@ makedumpfile nfs-utils python-rtslib-fb targetcli-fb -td-agent grub2 From 27d695b85e038af115f10367e47716b1a0aa3f2c Mon Sep 17 00:00:00 2001 From: John Wren Kennedy Date: Fri, 7 Aug 2020 08:34:30 -0600 Subject: [PATCH 23/39] Use gcc 8 for linux-pkg (#107) The 5.4 kernel headers define CONFIG_CC_HAS_ASM_INLINE, which prevents kernel modules from being compiled with gcc 7. --- lib/common.sh | 11 +++++++++++ setup.sh | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/lib/common.sh b/lib/common.sh index 325ba38..318fab4 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -796,3 +796,14 @@ function determine_target_kernels() { echo "Kernel versions to use to build modules:" echo " $KERNEL_VERSIONS" } + +# +# Install gcc 8, and make it the default +# +function install_gcc8() { + logmust install_pkgs gcc-8 g++-8 + logmust sudo update-alternatives --install /usr/bin/gcc gcc \ + /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7 + logmust sudo update-alternatives --install /usr/bin/gcc gcc \ + /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8 +} diff --git a/setup.sh b/setup.sh index b300088..f07223c 100755 --- a/setup.sh +++ b/setup.sh @@ -83,5 +83,12 @@ logmust install_pkgs \ logmust install_shfmt +# +# Starting with kernel 5.4, gcc 7 can no longer compile kernel modules, so +# install gcc 8 +# https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1849348 +# +logmust install_gcc8 + logmust git config --global user.email "eng@delphix.com" logmust git config --global user.name "Delphix Engineering" From 0739af313e2b2192c01b5076a1ff4bcc05f2f9b4 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Tue, 11 Aug 2020 16:39:14 -0700 Subject: [PATCH 24/39] Build and update kernel variants through linux-pkg (#97) = Motivation The Ubuntu kernels have served us well but we currently have two problems with them: 1] When we trigger a kernel bug we have to wait until a patch is released by Canonical. This may sound like a rare event but we've already hit bugs in iSCSI and NFS subsystems that we've had to work around. 2] There are certain kernel config options that we'd like to enable that won't be enabled by Canonical until later releases or at all even. Examples include BTF info in vmlinux and the PSI api needed by oomd. = This Patch Enable linux-pkg to compile the Ubuntu kernels from scratch. In addition it allows us to base our own patches on top of these kernels, and rebase them whenever we get new commits from the upstream Canonical kernels. This allows us to: 1] Fix upstream kernel bugs in a timely manner during customer and internal escalations. 2] Customize our kernel config as we see fit. 3] Have at least one standard workflow for experimenting with our own kernel changes that we may want to potentially upstream. --- default-package-config.sh | 266 ++++++++++++++++++++++++ packages/linux-kernel-aws/config.sh | 34 +++ packages/linux-kernel-azure/config.sh | 34 +++ packages/linux-kernel-gcp/config.sh | 34 +++ packages/linux-kernel-generic/config.sh | 40 ++++ packages/linux-kernel-oracle/config.sh | 34 +++ 6 files changed, 442 insertions(+) create mode 100644 packages/linux-kernel-aws/config.sh create mode 100644 packages/linux-kernel-azure/config.sh create mode 100644 packages/linux-kernel-gcp/config.sh create mode 100644 packages/linux-kernel-generic/config.sh create mode 100644 packages/linux-kernel-oracle/config.sh diff --git a/default-package-config.sh b/default-package-config.sh index 4c12a3d..94f9bce 100644 --- a/default-package-config.sh +++ b/default-package-config.sh @@ -34,3 +34,269 @@ function store_build_info() { echo "No build info available" >"$WORKDIR/build_info" fi } + +# +# The functions below are specific for the Linux kernel packages +# and contain the majority of their common code. +# +function kernel_prepare() { + logmust install_pkgs \ + equivs \ + devscripts \ + kernel-wedge +} + +# +# The configuration disabled below is specifically for uses +# of ${debian_rules_args}. Quoting the specific variable +# results in incorrect behavior and thus we disable that +# check. +# +# shellcheck disable=SC2086 +function kernel_build() { + local platform="$1" + # + # Note: Extra arguments can overwrite default arguments. + # For example in this function we default skipdbg + # to false, but if we pass "skipdbg=true" as an + # extra argument we will be overwriting this value + # to true. This is because when a variable's value + # is declared multiple times when invoking the + # debian/rules command, the rightmost declaration + # is the one that is actually used. + # + local debian_rules_extra_args="$2" + + logmust cd "$WORKDIR/repo" + + # + # We generate the default control file from Canonical + # so we can capture the ABI number (abinum) from + # Canonical's kernel - (see comment that follows for + # the reason and the relevant code for the logic). + # + logmust debian/rules debian/control + + # + # We overwrite the default abinum build variable with our + # version strings and at the same time retain the original + # abinum from Canonical by appending it at the end. + # + # We chose to mutate the abinum field as it is the least + # invasive for Ubuntu's build logic (e.g. most of the other + # variables actually interact with logic in the build). At + # the same time the abinum variable is part of the fields + # that we care about (e.g. package name, linux image file + # name, etc..). + # + # We still retain the original abinum by appending it at + # the end of the new one to maintain the mapping between + # Canonical's releases and our releases. + # + local canonical_abinum delphix_abinum + canonical_abinum=$(fakeroot debian/rules printenv | grep abinum | cut -d= -f2 | tr -d '[:space:]') + delphix_abinum="dlpx-$(date -u +"%Y%m%dt%H%M%S")-$(git rev-parse --short HEAD)-${canonical_abinum}" + + # + # skipdbg=false + # We need debug info for our debugging tools to work. + # Don't skip them. + # uefi_signed=false + # This variable defaults to true but since we don't have + # any intention and logic to provide signatures for now + # we set it to false to avoid any misconfigurations down + # the line. + # + local debian_rules_args="skipdbg=false uefi_signed=false abinum=${delphix_abinum} ${debian_rules_extra_args}" + + # + # Clean up everything generated so far and recreate the + # final control file with the arguments that we want. + # + logmust fakeroot debian/rules clean ${debian_rules_args} + + # + # Print the environment configuration solely for + # debugging purposes. + # + logmust fakeroot debian/rules printenv ${debian_rules_args} + + # + # The default value of the tool argument for mk-build-deps + # is the following: + # "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends" + # + # We append --yes to it to disable interactivity by apt-get + # and allow for automation. + # + local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" + logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}" + + logmust fakeroot debian/rules "binary-${platform}" ${debian_rules_args} + + logmust cd "$WORKDIR" + logmust mv ./*deb "artifacts/" +} + +# +# Syncing our kernel with the right upstream Canonical repo is not as +# straighforward as the other packages in linux-pkg. +# +# The Ubuntu developers maintain the timeline of the mainline kernel +# (kernel.org) in their git history. When it is time to use a new +# mainline version, they fork a new branch and then cherry-pick all their +# Ubuntu-specific generic patches on top of that and create their base +# tag. Then on top of that they cherry-pick their platform-specific +# patches (e.g. azure, aws, etc..) and create separate tags for each +# platform. This whole process is repeated for every bump in the kernel +# version (both mainline and ubuntu-specific). +# +# We want to track and sync our changes every time Canonical bumps +# their kernel version for the kernels that are used by the Delphix +# Engine that we release in order to stay up to date and lower our +# maintainance burden. As a result we do the following: +# +# * We have one kernel repo per platform. +# * Each of this repos is an Ubuntu kernel repo with our specific +# patches on top. +# * The vanilla Ubuntu kernel and our patches are divided by a single +# placeholder commit with the description "@@DELPHIX_PATCHSET_START@@". +# * Whenever the Ubuntu kernel version is bumped, we detect that +# change and use the new Ubuntu version as our base and cherry-pick +# the placeholder commit followed by our patches on top of it. +# +function kernel_update_upstream() { + local platform="$1" + + check_env UPSTREAM_GIT_URL + logmust cd "$WORKDIR/repo" + + # + # checkout our local branch that tracks upstream. + # + logmust git checkout -q upstream-HEAD + + # + # declare third-party upstream repository. + # + logmust git remote add upstream "$UPSTREAM_GIT_URL" + + # + # We get the kernel version and the ABI number from + # $_RET that is set from `get_kernel_from_platform()`. + # Example: + # + # $_RET -> 5.3.0-53-generic + # + # `cut -d '-' -f 1` of that -> 5.3.0 + # `cut -d '-' -f 3` of that -> 53 + # + # We need the kernel version and ABI number to figure + # out the latest upstream tag to sync with. + # + local kernel_version abinum + logmust get_kernel_for_platform "${platform}" + kernel_version=$(echo "$_RET" | cut -d '-' -f 1) + abinum=$(echo "$_RET" | cut -d '-' -f 2) + + # + # For each supported platform we will try to find the + # latest upstream tag to sync based on the kernel + # version and the ABI num that we got above. + # + # Note that "generic" (used mainly ESX) is a special + # case as we are currently using the HWE kernel image. + # + local tag_prefix + if [[ "${platform}" == generic ]] && + [[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then + tag_prefix="Ubuntu-hwe-${kernel_version}-${abinum}" + elif [[ "${platform}" == aws ]] || + [[ "${platform}" == azure ]] || + [[ "${platform}" == gcp ]] || + [[ "${platform}" == oracle ]]; then + + local kvers_major kvers_minor short_kvers + kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1) + kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2) + short_kvers="${kvers_major}.${kvers_minor}" + + tag_prefix="Ubuntu-${platform}-${short_kvers}-${kernel_version}-${abinum}" + else + die "assertion: unexpected platform: ${platform}" + fi + echo "note: upstream tag prefix used: ${tag_prefix}" + + # + # Query for upstream tag info based on the prefix that we've + # assembled. + # + # = Why the `tail -n 1` part? + # + # Using `git ls-remote` and `grep` with the tag's prefix alone + # may sometimes return two (and theoretically more?) results + # due to Ubuntu's "point releases". Point releases are specific + # to LTS releases and more info about them can be found in the + # links below: + # [1] https://wiki.ubuntu.com/LTS + # [2] https://wiki.ubuntu.com/PointReleaseProcess + # + # Example: + # ``` + # $ git ls-remote --tags upstream | grep Ubuntu-oracle-5.3-5.3.0-1015 + # df8fd7d8802d59 refs/tags/Ubuntu-oracle-5.3-5.3.0-1015.16_18.04.1 + # 0fe5cd29e90a5e refs/tags/Ubuntu-oracle-5.3-5.3.0-1015.16_18.04.2 + # ``` + # + # We most probably want the latest point release of a specific + # kernel thus we add `tail -n 1` in the pipeline below. + # + local upstream_tag_info + upstream_tag_info=$(git ls-remote --tags --ref upstream | grep "${tag_prefix}" | tail -n 1) + [[ -z "${upstream_tag_info}" ]] && die "could not find upstream tag for tag prefix: ${tag_prefix}" + + local upstream_tag + upstream_tag=$(echo "${upstream_tag_info}" | awk -F / '{print $3}') + [[ -z "${upstream_tag}" ]] && die "could not extract upstream tag name from the tag info" + echo "note: upstream tag: ${upstream_tag}" + logmust git fetch upstream "${upstream_tag}" + + # + # Ensure that there is a commit marking the start of + # the Delphix set of patches. Then get the hash of + # the commit right before it. + # + local dlpx_patch_end dlpx_patch_start current_ubuntu_commit + dlpx_patch_start=$(git log --pretty=oneline repo-HEAD | grep @@DELPHIX_PATCHSET_START@@ | awk '{ print $1 }') + [[ -z "${dlpx_patch_start}" ]] && die "could not find DELPHIX_PATCHSET_START" + [[ $(wc -l <<<"${dlpx_patch_start}") != 1 ]] && die "multiple DELPHIX_PATCHSET_START commits - ${dlpx_patch_start}" + current_ubuntu_commit=$(git rev-parse "${dlpx_patch_start}"^) + [[ -z "${current_ubuntu_commit}" ]] && die "could not find commit before DELPHIX_PATCHSET_START" + dlpx_patch_end=$(git rev-parse repo-HEAD) + [[ -z "${dlpx_patch_end}" ]] && die "could not find repo-HEAD's head commit" + + # + # Compare that commit with the head commit of the + # upstream tag. If the commits are the same then + # there is nothing for us to do as we are using + # the most up-to-date tag as the base for our set + # of patches. On the other hand, if the commits + # differ then it means that the upstream has been + # updated, at which point we need to cherry-pick + # our patches on top of the new upstream. + # + local upstream_head_commit + upstream_head_commit=$(git rev-parse upstream-HEAD) + [[ -z "${upstream_head_commit}" ]] && die "could not find upstream-HEAD's head commit" + + if [[ "${current_ubuntu_commit}" == "${upstream_head_commit}" ]]; then + echo "NOTE: upstream for $PACKAGE is already up-to-date." + else + # shellcheck disable=SC2086 + logmust git cherry-pick ${dlpx_patch_start}^..${dlpx_patch_end} + + logmust touch "$WORKDIR/upstream-updated" + fi + + logmust cd "$WORKDIR" +} diff --git a/packages/linux-kernel-aws/config.sh b/packages/linux-kernel-aws/config.sh new file mode 100644 index 0000000..b42417b --- /dev/null +++ b/packages/linux-kernel-aws/config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-aws.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-aws/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + kernel_build "aws" +} + +function update_upstream() { + kernel_update_upstream "aws" +} diff --git a/packages/linux-kernel-azure/config.sh b/packages/linux-kernel-azure/config.sh new file mode 100644 index 0000000..ddb8c34 --- /dev/null +++ b/packages/linux-kernel-azure/config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-azure.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-azure/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + kernel_build "azure" +} + +function update_upstream() { + kernel_update_upstream "azure" +} diff --git a/packages/linux-kernel-gcp/config.sh b/packages/linux-kernel-gcp/config.sh new file mode 100644 index 0000000..2ba303b --- /dev/null +++ b/packages/linux-kernel-gcp/config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-gcp.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-gcp/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + kernel_build "gcp" +} + +function update_upstream() { + kernel_update_upstream "gcp" +} diff --git a/packages/linux-kernel-generic/config.sh b/packages/linux-kernel-generic/config.sh new file mode 100644 index 0000000..baa0708 --- /dev/null +++ b/packages/linux-kernel-generic/config.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-generic.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + # + # flavours=generic + # By default the generic kernel variant from Canonical + # builds both the generic and the low-latency kernel. + # We don't care about the latter. + # + kernel_build "generic" "flavours=generic" +} + +function update_upstream() { + kernel_update_upstream "generic" +} diff --git a/packages/linux-kernel-oracle/config.sh b/packages/linux-kernel-oracle/config.sh new file mode 100644 index 0000000..85b1a10 --- /dev/null +++ b/packages/linux-kernel-oracle/config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-oracle.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oracle/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + kernel_build "oracle" +} + +function update_upstream() { + kernel_update_upstream "oracle" +} From 65df73e5453cc20f2c44d9c7fc994ebccdafd92a Mon Sep 17 00:00:00 2001 From: smeda-delphix <69540224+smeda-delphix@users.noreply.github.com> Date: Wed, 12 Aug 2020 21:58:19 -0400 Subject: [PATCH 25/39] Add check to verify copyright file exists for all platform packages. (#109) --- README.md | 11 +++++++++++ buildpkg.sh | 3 +++ default-package-config.sh | 20 ++++++++++++++++++++ packages/adoptopenjdk/config.sh | 1 + packages/delphix-sso-app/config.sh | 1 + packages/java8/config.sh | 1 + 6 files changed, 37 insertions(+) diff --git a/README.md b/README.md index dc63462..3bd33d7 100644 --- a/README.md +++ b/README.md @@ -409,6 +409,17 @@ package's source. `build_info` files for each package are consumed by the [metapackage](./build-info-pkg) when running [buildlist.sh](#buildlistsh). +#### Post Build Checks + +The `post_build_checks()` hook is optional. It is responsible for checking if +the debian package being built has copyright file associated with it in the +right location. This file will be used to generate the license information for +the appliance. + +A default hook is provided in `default-package-config.sh` and will +be used if it is not overriden. The packages that are eligible to skip this check +should define `SKIP_COPYRIGHTS_CHECK=true` in their respective `config.sh` files. + #### Checkstyle The `checkstyle()` hook is optional. It is called before building the package if diff --git a/buildpkg.sh b/buildpkg.sh index 76d70d4..edd3560 100755 --- a/buildpkg.sh +++ b/buildpkg.sh @@ -240,6 +240,9 @@ logmust rm "$WORKDIR/building" logmust cd "$WORKDIR" stage store_build_info +logmust cd "$WORKDIR" +stage post_build_checks + echo_success "Package $PACKAGE has been built successfully." echo "Build products are in $WORKDIR/artifacts" echo "" diff --git a/default-package-config.sh b/default-package-config.sh index 94f9bce..a24685d 100644 --- a/default-package-config.sh +++ b/default-package-config.sh @@ -300,3 +300,23 @@ function kernel_update_upstream() { logmust cd "$WORKDIR" } + +function post_build_checks() { + + # This function checks for SKIP_COPYRIGHTS_CHECK flag + # in config.sh file of each package. If the flag is + # present and is set to 'true', the check will be skipped. + # The license information for the platform packages are + # generated based on Ubuntu package convention and are + # picked from copyright file under /debian folder. As a + # part of the check we look for existance of the file in + # each package. + if [[ "$SKIP_COPYRIGHTS_CHECK" != true ]]; then + echo "Start copyright check" + file_count=$(find "$WORKDIR/repo" | grep 'debian/copyright' -c) + + if [[ ! $file_count -gt 0 ]]; then + logmust die "Copyright file is missing in the package repository." + fi + fi +} diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh index aae84c1..6902606 100644 --- a/packages/adoptopenjdk/config.sh +++ b/packages/adoptopenjdk/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SKIP_COPYRIGHTS_CHECK=true tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u212b04.tar.gz" jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 8c47e86..b3a651e 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/saml-app.git" JDK_PATH_FILE="$TOP/packages/adoptopenjdk/tmp/artifacts/JDK_PATH" +SKIP_COPYRIGHTS_CHECK=true function prepare() { java_package_exists=$(dpkg-query --show adoptopenjdk-java8-jdk >/dev/null 2>&1) diff --git a/packages/java8/config.sh b/packages/java8/config.sh index 39f98d3..d583622 100644 --- a/packages/java8/config.sh +++ b/packages/java8/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SKIP_COPYRIGHTS_CHECK=true tarfile="jdk-8u171-linux-x64.tar.gz" jdk_path="/usr/lib/jvm/oracle-java8-jdk-amd64" From e5befe56e612a4489d3eb7c1f2239c2e927b476a Mon Sep 17 00:00:00 2001 From: smeda-delphix <69540224+smeda-delphix@users.noreply.github.com> Date: Thu, 13 Aug 2020 12:14:52 -0400 Subject: [PATCH 26/39] Add skip copyright check flag for td-agent prebuilt package (#111) --- packages/td-agent-prebuilt/config.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/td-agent-prebuilt/config.sh b/packages/td-agent-prebuilt/config.sh index d21b7ac..4fef2cc 100644 --- a/packages/td-agent-prebuilt/config.sh +++ b/packages/td-agent-prebuilt/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SKIP_COPYRIGHTS_CHECK=true package="td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb" function fetch() { From 927e206506787fc136b02fc33933595b8378a3b0 Mon Sep 17 00:00:00 2001 From: smeda-delphix <69540224+smeda-delphix@users.noreply.github.com> Date: Mon, 17 Aug 2020 15:11:01 -0400 Subject: [PATCH 27/39] update copyright check script (#114) --- default-package-config.sh | 16 +++++++++------- packages/adoptopenjdk/config.sh | 1 - packages/crypt-blowfish/config.sh | 1 + packages/delphix-sso-app/config.sh | 1 - packages/grub2/config.sh | 2 ++ packages/java8/config.sh | 1 - 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/default-package-config.sh b/default-package-config.sh index a24685d..442d325 100644 --- a/default-package-config.sh +++ b/default-package-config.sh @@ -308,15 +308,17 @@ function post_build_checks() { # present and is set to 'true', the check will be skipped. # The license information for the platform packages are # generated based on Ubuntu package convention and are - # picked from copyright file under /debian folder. As a + # picked from copyright file in debian package. As a # part of the check we look for existance of the file in # each package. if [[ "$SKIP_COPYRIGHTS_CHECK" != true ]]; then - echo "Start copyright check" - file_count=$(find "$WORKDIR/repo" | grep 'debian/copyright' -c) - - if [[ ! $file_count -gt 0 ]]; then - logmust die "Copyright file is missing in the package repository." - fi + cd "$WORKDIR/artifacts" || return + + set -o pipefail + for deb in *.deb; do + echo "Running: dpkg-deb -c $deb | grep '/usr/share/doc/' | grep copyright" + dpkg-deb -c "$deb" | grep '/usr/share/doc/' | grep copyright || die "copyright file missing for package $deb" + done + set +o pipefail fi } diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh index 6902606..aae84c1 100644 --- a/packages/adoptopenjdk/config.sh +++ b/packages/adoptopenjdk/config.sh @@ -17,7 +17,6 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none -SKIP_COPYRIGHTS_CHECK=true tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u212b04.tar.gz" jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" diff --git a/packages/crypt-blowfish/config.sh b/packages/crypt-blowfish/config.sh index e512ddc..1f020ad 100644 --- a/packages/crypt-blowfish/config.sh +++ b/packages/crypt-blowfish/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/os-platform/linux-3rd-party-pkgs.git" DEFAULT_PACKAGE_VERSION="1.0.0" +SKIP_COPYRIGHTS_CHECK=true function build() { logmust cd "$WORKDIR/repo/crypt_blowfish" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index b3a651e..8c47e86 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -18,7 +18,6 @@ DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/saml-app.git" JDK_PATH_FILE="$TOP/packages/adoptopenjdk/tmp/artifacts/JDK_PATH" -SKIP_COPYRIGHTS_CHECK=true function prepare() { java_package_exists=$(dpkg-query --show adoptopenjdk-java8-jdk >/dev/null 2>&1) diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh index 0fa91ff..12decdd 100644 --- a/packages/grub2/config.sh +++ b/packages/grub2/config.sh @@ -21,6 +21,8 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/grub2" UPSTREAM_GIT_URL=https://git.launchpad.net/ubuntu/+source/grub2 UPSTREAM_GIT_BRANCH=applied/ubuntu/bionic-updates +SKIP_COPYRIGHTS_CHECK=true + # # Install build dependencies for the package. # diff --git a/packages/java8/config.sh b/packages/java8/config.sh index d583622..39f98d3 100644 --- a/packages/java8/config.sh +++ b/packages/java8/config.sh @@ -17,7 +17,6 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none -SKIP_COPYRIGHTS_CHECK=true tarfile="jdk-8u171-linux-x64.tar.gz" jdk_path="/usr/lib/jvm/oracle-java8-jdk-amd64" From 493a320b90e6ccea7b7fc2dc1051adb760819bbf Mon Sep 17 00:00:00 2001 From: Derek Woodham <69479759+derek-woodham-delphix@users.noreply.github.com> Date: Tue, 18 Aug 2020 16:09:29 -0400 Subject: [PATCH 28/39] Upgrade java to 8u262b10 (#115) --- packages/adoptopenjdk/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 packages/adoptopenjdk/config.sh diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh old mode 100644 new mode 100755 index aae84c1..df35453 --- a/packages/adoptopenjdk/config.sh +++ b/packages/adoptopenjdk/config.sh @@ -17,7 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none -tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u212b04.tar.gz" +tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz" jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" function prepare() { From 375bc36cc3aad40e13e4f79467f17ef119d75d7b Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Tue, 1 Sep 2020 09:07:37 -0700 Subject: [PATCH 29/39] Add savedump userland package (#120) --- package-lists/build/userland.pkgs | 1 + packages/savedump/config.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 packages/savedump/config.sh diff --git a/package-lists/build/userland.pkgs b/package-lists/build/userland.pkgs index 061e074..0220d2f 100644 --- a/package-lists/build/userland.pkgs +++ b/package-lists/build/userland.pkgs @@ -25,6 +25,7 @@ gdb-python makedumpfile nfs-utils python-rtslib-fb +savedump sdb targetcli-fb performance-diagnostics diff --git a/packages/savedump/config.sh b/packages/savedump/config.sh new file mode 100644 index 0000000..2b1fec9 --- /dev/null +++ b/packages/savedump/config.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/savedump.git" +DEFAULT_PACKAGE_VERSION=1.0.0 + +function prepare() { + logmust install_pkgs \ + git \ + python3-distutils +} + +function build() { + logmust dpkg_buildpackage_default +} From 0512edebbba89eb6314daa116912c1496e42e099 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 3 Sep 2020 12:24:54 -0400 Subject: [PATCH 30/39] TOOL-10140 linux-pkg: add new build lists and new package dependencies field (#117) --- .github/scripts/verify-query-packages.sh | 31 ++++++++++++++-- lib/common.sh | 47 ++++++++++++++++++++++++ package-lists/build/kernel-modules.pkgs | 8 ++++ package-lists/build/main.pkgs | 29 +++++++++++++++ package-lists/update/main.pkgs | 27 ++++++++++++++ packages/adoptopenjdk/config.sh | 2 + packages/bpftrace/config.sh | 1 + packages/connstat/config.sh | 1 + packages/delphix-kernel/config.sh | 1 + packages/delphix-sso-app/config.sh | 1 + packages/drgn/config.sh | 1 + packages/grub2/config.sh | 1 + packages/recovery-environment/config.sh | 1 + packages/targetcli-fb/config.sh | 1 + packages/zfs/config.sh | 1 + query-packages.sh | 39 +++++++++++++++++--- 16 files changed, 184 insertions(+), 8 deletions(-) create mode 100644 package-lists/build/kernel-modules.pkgs create mode 100644 package-lists/build/main.pkgs create mode 100644 package-lists/update/main.pkgs diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index c795a16..e66171f 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -30,12 +30,21 @@ read -r -a fields <<<"$(./query-packages.sh single -o git-url zfs 2>&1)" test ${#fields[@]} -eq 1 test "${fields[0]}" == 'https://github.com/delphix/zfs.git' +# Expect: "bpftrace bcc true https://github.com/delphix/bpftrace.git" +read -r -a fields <<<"$(./query-packages.sh single -o name,dependencies,can-update,git-url bpftrace 2>&1)" +test ${#fields[@]} -eq 4 +test "${fields[0]}" == 'bpftrace' +test "${fields[1]}" == 'bcc' +test "${fields[2]}" == 'true' +test "${fields[3]}" == 'https://github.com/delphix/bpftrace.git' + # Expect that "list all" outputs all directory names under packages/ diff <(ls -1 packages | sort) <(./query-packages.sh list all 2>&1 | sort) -# Expect that outputing git-url for all packages works and that the output +# Expect that outputing dependencies & git-url for all packages works and that the output # length corresponds to the number of packages. -test "$(ls -1 packages | wc -l)" -eq "$(./query-packages.sh list -o name,git-url all 2>&1 | wc -l)" +test "$(ls -1 packages | wc -l)" -eq \ + "$(./query-packages.sh list -o name,dependencies,can-update,git-url all 2>&1 | wc -l)" # Check that all package lists under package-lists\ can be loaded and that each # line of the output of the command actually refers to a package. @@ -47,9 +56,25 @@ find package-lists -name '*.pkgs' | while read -r list; do ) >/dev/null done -# Check that querying "appliance" list works +# Check that querying the built-in "appliance" list works ./query-packages.sh list appliance >/dev/null +# Check that querying lists used by the Delphix build works. +./query-packages.sh list build/main.pkgs >/dev/null +./query-packages.sh list build/kernel-modules.pkgs >/dev/null +./query-packages.sh list linux-kernel >/dev/null +./query-packages.sh list update/main.pkgs >/dev/null + +# Check that overriding TARGET_KERNEL_FLAVORS changes which kernel packages are +# returned. +test "$(./query-packages.sh list linux-kernel | wc -l | awk '{print $1}')" -gt 1 +test "$(TARGET_KERNEL_FLAVORS=generic ./query-packages.sh list linux-kernel)" == "linux-kernel-generic" + +# Check that when a package has multiple dependencies they are printed in the +# expected format. +test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o dependencies zfs)" == \ + "linux-kernel-generic,linux-kernel-aws" + # Check that the output from the appliance list contains zfs and # delphix-platform packages. Note, we explicitly do not use grep -q here as it # exits as soon as a match is found and that causes a broken pipe error as diff --git a/lib/common.sh b/lib/common.sh index 318fab4..8cc6658 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -21,6 +21,7 @@ export DEBIAN_FRONTEND=noninteractive # TODO: allow updating upstream for other branches than master export REPO_UPSTREAM_BRANCH="upstreams/master" +export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle" export UBUNTU_DISTRIBUTION="bionic" @@ -221,12 +222,14 @@ function reset_package_config_variables() { DEFAULT_PACKAGE_GIT_BRANCH DEFAULT_PACKAGE_GIT_VERSION DEFAULT_PACKAGE_GIT_REVISION + PACKAGE_DEPENDENCIES UPSTREAM_SOURCE_PACKAGE UPSTREAM_GIT_URL UPSTREAM_GIT_BRANCH WORKDIR PKGDIR PACKAGE_PREFIX + SKIP_COPYRIGHTS_CHECK " for var in $vars; do @@ -300,6 +303,24 @@ function load_package_config() { die "$PACKAGE: DEFAULT_PACKAGE_GIT_URL must begin with " \ "https:// or be set to 'none'" + local dependency + local deps_array=() + for dependency in $PACKAGE_DEPENDENCIES; do + # + # Check for special value @linux-kernel which resolves to + # all flavors of linux kernel packages. + # + if [[ $dependency == '@linux-kernel' ]]; then + logmust list_linux_kernel_packages + deps_array+=("${_RET_LIST[@]}") + continue + fi + (check_package_exists "$dependency") || + die "Invalid package dependency '$dependency'" + deps_array+=("$dependency") + done + PACKAGE_DEPENDENCIES="${deps_array[*]}" + # # Check for variables related to update_upstream() hook # @@ -501,6 +522,32 @@ function get_package_list_file() { fi } +# +# List all target kernel packages. By default, it returns all the kernel +# flavors supported and built by linux-pkg, however this can be overridden +# via TARGET_KERNEL_FLAVORS, which can be useful when testing changes against +# a specific, or even mainline kernel. +# +function list_linux_kernel_packages() { + local kernel + + _RET_LIST=() + if [[ -n "$TARGET_KERNEL_FLAVORS" ]]; then + for kernel in $TARGET_KERNEL_FLAVORS; do + (check_package_exists "linux-kernel-$kernel") || + die "Invalid entry '$kernel' in TARGET_KERNEL_FLAVORS" + _RET_LIST+=("linux-kernel-$kernel") + done + else + for kernel in $SUPPORTED_KERNEL_FLAVORS; do + check_package_exists "linux-kernel-$kernel" + _RET_LIST+=("linux-kernel-$kernel") + done + fi + + return 0 +} + function install_shfmt() { if [[ ! -f /usr/local/bin/shfmt ]]; then logmust sudo wget -nv -O /usr/local/bin/shfmt \ diff --git a/package-lists/build/kernel-modules.pkgs b/package-lists/build/kernel-modules.pkgs new file mode 100644 index 0000000..56b0eb3 --- /dev/null +++ b/package-lists/build/kernel-modules.pkgs @@ -0,0 +1,8 @@ +# +# List of kernel packages to be included in the Delphix Appliance. +# This list should not include the linux kernel packages themselves. +# + +connstat +delphix-kernel +zfs diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs new file mode 100644 index 0000000..28bffca --- /dev/null +++ b/package-lists/build/main.pkgs @@ -0,0 +1,29 @@ +# +# List of non-kernel packages to be included in the Delphix Appliance. +# + +adoptopenjdk +bcc +bpftrace +challenge-response +cloud-init +crash +crash-python +crypt-blowfish +delphix-platform +delphix-sso-app +drgn +gdb-python +grub2 +libkdumpfile +make-jpkg +makedumpfile +nfs-utils +performance-diagnostics +ptools +python-rtslib-fb +recovery-environment +savedump +sdb +targetcli-fb +td-agent-prebuilt diff --git a/package-lists/update/main.pkgs b/package-lists/update/main.pkgs new file mode 100644 index 0000000..73ff7c3 --- /dev/null +++ b/package-lists/update/main.pkgs @@ -0,0 +1,27 @@ +# +# All those packages will be auto-updated by the linux-pkg-update/userland job. +# Note that if you want to keep a package's upstream branch to be updated +# but prevent it from being merged, add the package to +# auto-merge-blacklist.pkg. +# + +# Note: we do not auto-update bcc for now as upstream tends to introduce +# changes that break bpftrace. Instead, we manually update to tagged versions. +# In the future we may want to automatically determine the latest tag and +# auto-update to that. +#bcc + +bpftrace +cloud-init +crash +drgn +grub2 +libkdumpfile +makedumpfile +nfs-utils +python-rtslib-fb +targetcli-fb + +# Do not attempt to auto-update td-agent for now as breaking changes were +# introduced upstream. +#td-agent diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh index df35453..84eab59 100755 --- a/packages/adoptopenjdk/config.sh +++ b/packages/adoptopenjdk/config.sh @@ -17,6 +17,8 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +PACKAGE_DEPENDENCIES="make-jpkg" + tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz" jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" diff --git a/packages/bpftrace/config.sh b/packages/bpftrace/config.sh index 95a1766..ef2f636 100644 --- a/packages/bpftrace/config.sh +++ b/packages/bpftrace/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bpftrace.git" DEFAULT_PACKAGE_VERSION=1.0.0 +PACKAGE_DEPENDENCIES="bcc" UPSTREAM_GIT_URL="https://github.com/iovisor/bpftrace.git" UPSTREAM_GIT_BRANCH="master" diff --git a/packages/connstat/config.sh b/packages/connstat/config.sh index 6fbb86a..8bb5e3a 100644 --- a/packages/connstat/config.sh +++ b/packages/connstat/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/connstat.git" DEFAULT_PACKAGE_VERSION="1.0.0" +PACKAGE_DEPENDENCIES="@linux-kernel" function prepare() { logmust install_pkgs \ diff --git a/packages/delphix-kernel/config.sh b/packages/delphix-kernel/config.sh index 2014156..f464dfd 100644 --- a/packages/delphix-kernel/config.sh +++ b/packages/delphix-kernel/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-kernel.git" DEFAULT_PACKAGE_VERSION="1.0.0" +PACKAGE_DEPENDENCIES="@linux-kernel" function prepare() { logmust install_pkgs \ diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 8c47e86..956fdb6 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/saml-app.git" JDK_PATH_FILE="$TOP/packages/adoptopenjdk/tmp/artifacts/JDK_PATH" +PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { java_package_exists=$(dpkg-query --show adoptopenjdk-java8-jdk >/dev/null 2>&1) diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index c5b94f6..737263a 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -18,6 +18,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/drgn.git" DEFAULT_PACKAGE_VERSION=1.0.0 +PACKAGE_DEPENDENCIES="libkdumpfile" UPSTREAM_GIT_URL="https://github.com/osandov/drgn.git" UPSTREAM_GIT_BRANCH="master" diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh index 12decdd..254fe76 100644 --- a/packages/grub2/config.sh +++ b/packages/grub2/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/grub2" +PACKAGE_DEPENDENCIES="zfs" UPSTREAM_GIT_URL=https://git.launchpad.net/ubuntu/+source/grub2 UPSTREAM_GIT_BRANCH=applied/ubuntu/bionic-updates diff --git a/packages/recovery-environment/config.sh b/packages/recovery-environment/config.sh index b5b0bf5..1ca3394 100644 --- a/packages/recovery-environment/config.sh +++ b/packages/recovery-environment/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/recovery-environment.git" DEFAULT_PACKAGE_VERSION=1.0.0 +PACKAGE_DEPENDENCIES="zfs" ZFS_DEB_PATH="$TOP/packages/zfs/tmp/artifacts" function prepare() { diff --git a/packages/targetcli-fb/config.sh b/packages/targetcli-fb/config.sh index 68f2ef7..7b6f406 100644 --- a/packages/targetcli-fb/config.sh +++ b/packages/targetcli-fb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/targetcli-fb.git" +PACKAGE_DEPENDENCIES="python-rtslib-fb" # Note: we get the package version programatically in build() UPSTREAM_SOURCE_PACKAGE=targetcli-fb diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index 2d02fc2..6d3f695 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" DEFAULT_PACKAGE_VERSION="0.8.0" +PACKAGE_DEPENDENCIES="@linux-kernel" UPSTREAM_GIT_URL="https://github.com/zfsonlinux/zfs.git" UPSTREAM_GIT_BRANCH="master" diff --git a/query-packages.sh b/query-packages.sh index 14fe386..b72f81a 100755 --- a/query-packages.sh +++ b/query-packages.sh @@ -19,7 +19,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 TOP="$PWD" source "$TOP/lib/common.sh" -ALL_OUTPUT_FIELDS=(name git-url dependencies) +ALL_OUTPUT_FIELDS=(name git-url dependencies can-update) function usage() { local output_fields="${ALL_OUTPUT_FIELDS[*]}" @@ -38,6 +38,7 @@ function usage() { echo " special values to the list command:" echo " all: displays all known packages" echo " appliance: displays all packages used by appliance" + echo " linux-kernel: displays all linux kernel packages" echo "" echo " -o Comma delimited output fields." echo " Possible values: ${output_fields// /, }." @@ -50,19 +51,43 @@ function usage() { function print_package() { local pkgname="$1" local outarray=() + local deps_array=() + # + # Run this in a sub-shell so that functions and variables sourced from + # load_package_config() are reset each time we load a new package. + # ( local field + local dep load_package_config "$pkgname" >/dev/null for field in "${ACTIVE_OUTPUT_FIELDS[@]}"; do case "$field" in name) outarray+=("$pkgname") ;; git-url) outarray+=("${DEFAULT_PACKAGE_GIT_URL:-none}") ;; - dependencies) outarray+=(none) ;; + dependencies) + for dep in $PACKAGE_DEPENDENCIES; do + check_package_exists "$dep" + deps_array+=("$dep") + done + if [[ ${#deps_array[@]} -eq 0 ]]; then + outarray+=(none) + else + # comma-separated list of dependencies + outarray+=("$( + IFS=, + echo "${deps_array[*]}" + )") + fi + ;; + can-update) + type -t "update_upstream" >/dev/null && + outarray+=(true) || outarray+=(false) + ;; esac done IFS=$'\t' echo "${outarray[*]}" - ) + ) || die "Failed to print info for package '$pkgname'" } function query_list() { @@ -77,14 +102,18 @@ function query_list() { list_all_packages >/dev/null elif [[ $list == appliance ]]; then # concatenate kernel and userland packages - read_package_list "$TOP/package-lists/build/kernel.pkgs" >/dev/null + read_package_list "$TOP/package-lists/build/kernel-modules.pkgs" >/dev/null local kernel_list=("${_RET_LIST[@]}") - read_package_list "$TOP/package-lists/build/userland.pkgs" >/dev/null + list_linux_kernel_packages >/dev/null + kernel_list+=("${_RET_LIST[@]}") + read_package_list "$TOP/package-lists/build/main.pkgs" >/dev/null _RET_LIST+=("${kernel_list[@]}") # check that there are no duplicates dups=$(printf '%s\n' "${_RET_LIST[@]}" | sort | uniq -d) [[ -z $dups ]] || die "Some apliance packages appear in both" \ "build/kernel.pkgs and build/userland.pkgs:\\n${dups}" + elif [[ $list == linux-kernel ]]; then + list_linux_kernel_packages >/dev/null else read_package_list "$TOP/package-lists/${list}" >/dev/null fi From 7e7fb0a27cadf85bc1c7571d4017cb27b3a94dc0 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 12 Feb 2020 10:24:41 -0500 Subject: [PATCH 31/39] DLPX-68435 crash dump fails on 5.3 kernel Enable auto-update for the makedumpfile package and do not remove kdump-tools to keep both packages built from the same source. --- package-lists/update/userland.pkgs | 1 + packages/makedumpfile/config.sh | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs index 5d0fa17..f0bcbaa 100644 --- a/package-lists/update/userland.pkgs +++ b/package-lists/update/userland.pkgs @@ -16,6 +16,7 @@ cloud-init crash drgn libkdumpfile +makedumpfile nfs-utils python-rtslib-fb targetcli-fb diff --git a/packages/makedumpfile/config.sh b/packages/makedumpfile/config.sh index d8b5343..03676b3 100644 --- a/packages/makedumpfile/config.sh +++ b/packages/makedumpfile/config.sh @@ -28,11 +28,9 @@ function prepare() { function build() { logmust cd "$WORKDIR/repo" if [[ -z "$PACKAGE_VERSION" ]]; then - logmust eval PACKAGE_VERSION="$(grep '^VERSION=' Makefile | cut -d "=" -f 2)" + logmust eval PACKAGE_VERSION="1:$(grep '^VERSION=' Makefile | cut -d "=" -f 2)" fi logmust dpkg_buildpackage_default - # we only need makedumpfile package - logmust rm -f "$WORKDIR/artifacts/"kdump-tools_*_amd64.deb } function update_upstream() { From affe2f46a458a8a0c3fdfd233e4ca46a6e9b0280 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 13 May 2020 11:07:48 -0700 Subject: [PATCH 32/39] delphix-platform: "make check" is no longer supported (#94) --- packages/delphix-platform/config.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/delphix-platform/config.sh b/packages/delphix-platform/config.sh index 1e4baba..69a6cc0 100644 --- a/packages/delphix-platform/config.sh +++ b/packages/delphix-platform/config.sh @@ -19,12 +19,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-platform.git" DEFAULT_PACKAGE_VERSION="1.0.0" -function checkstyle() { - logmust cd "$WORKDIR/repo" - logmust ansible-playbook bootstrap/playbook.yml - logmust ./scripts/docker-run.sh make check -} - function build() { logmust cd "$WORKDIR/repo" logmust ansible-playbook bootstrap/playbook.yml From ca018d092818ccdcb4d25330ebc18ab6ad59c19b Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 30 Sep 2020 13:56:49 -0400 Subject: [PATCH 33/39] Build and update kernel variants through linux-pkg (#97) = Motivation The Ubuntu kernels have served us well but we currently have two problems with them: 1] When we trigger a kernel bug we have to wait until a patch is released by Canonical. This may sound like a rare event but we've already hit bugs in iSCSI and NFS subsystems that we've had to work around. 2] There are certain kernel config options that we'd like to enable that won't be enabled by Canonical until later releases or at all even. Examples include BTF info in vmlinux and the PSI api needed by oomd. = This Patch Enable linux-pkg to compile the Ubuntu kernels from scratch. In addition it allows us to base our own patches on top of these kernels, and rebase them whenever we get new commits from the upstream Canonical kernels. This allows us to: 1] Fix upstream kernel bugs in a timely manner during customer and internal escalations. 2] Customize our kernel config as we see fit. 3] Have at least one standard workflow for experimenting with our own kernel changes that we may want to potentially upstream. --- default-package-config.sh | 266 ++++++++++++++++++++++++ packages/linux-kernel-aws/config.sh | 34 +++ packages/linux-kernel-azure/config.sh | 34 +++ packages/linux-kernel-gcp/config.sh | 34 +++ packages/linux-kernel-generic/config.sh | 40 ++++ packages/linux-kernel-oracle/config.sh | 34 +++ 6 files changed, 442 insertions(+) create mode 100644 packages/linux-kernel-aws/config.sh create mode 100644 packages/linux-kernel-azure/config.sh create mode 100644 packages/linux-kernel-gcp/config.sh create mode 100644 packages/linux-kernel-generic/config.sh create mode 100644 packages/linux-kernel-oracle/config.sh diff --git a/default-package-config.sh b/default-package-config.sh index 1c5009c..442d325 100644 --- a/default-package-config.sh +++ b/default-package-config.sh @@ -35,6 +35,272 @@ function store_build_info() { fi } +# +# The functions below are specific for the Linux kernel packages +# and contain the majority of their common code. +# +function kernel_prepare() { + logmust install_pkgs \ + equivs \ + devscripts \ + kernel-wedge +} + +# +# The configuration disabled below is specifically for uses +# of ${debian_rules_args}. Quoting the specific variable +# results in incorrect behavior and thus we disable that +# check. +# +# shellcheck disable=SC2086 +function kernel_build() { + local platform="$1" + # + # Note: Extra arguments can overwrite default arguments. + # For example in this function we default skipdbg + # to false, but if we pass "skipdbg=true" as an + # extra argument we will be overwriting this value + # to true. This is because when a variable's value + # is declared multiple times when invoking the + # debian/rules command, the rightmost declaration + # is the one that is actually used. + # + local debian_rules_extra_args="$2" + + logmust cd "$WORKDIR/repo" + + # + # We generate the default control file from Canonical + # so we can capture the ABI number (abinum) from + # Canonical's kernel - (see comment that follows for + # the reason and the relevant code for the logic). + # + logmust debian/rules debian/control + + # + # We overwrite the default abinum build variable with our + # version strings and at the same time retain the original + # abinum from Canonical by appending it at the end. + # + # We chose to mutate the abinum field as it is the least + # invasive for Ubuntu's build logic (e.g. most of the other + # variables actually interact with logic in the build). At + # the same time the abinum variable is part of the fields + # that we care about (e.g. package name, linux image file + # name, etc..). + # + # We still retain the original abinum by appending it at + # the end of the new one to maintain the mapping between + # Canonical's releases and our releases. + # + local canonical_abinum delphix_abinum + canonical_abinum=$(fakeroot debian/rules printenv | grep abinum | cut -d= -f2 | tr -d '[:space:]') + delphix_abinum="dlpx-$(date -u +"%Y%m%dt%H%M%S")-$(git rev-parse --short HEAD)-${canonical_abinum}" + + # + # skipdbg=false + # We need debug info for our debugging tools to work. + # Don't skip them. + # uefi_signed=false + # This variable defaults to true but since we don't have + # any intention and logic to provide signatures for now + # we set it to false to avoid any misconfigurations down + # the line. + # + local debian_rules_args="skipdbg=false uefi_signed=false abinum=${delphix_abinum} ${debian_rules_extra_args}" + + # + # Clean up everything generated so far and recreate the + # final control file with the arguments that we want. + # + logmust fakeroot debian/rules clean ${debian_rules_args} + + # + # Print the environment configuration solely for + # debugging purposes. + # + logmust fakeroot debian/rules printenv ${debian_rules_args} + + # + # The default value of the tool argument for mk-build-deps + # is the following: + # "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends" + # + # We append --yes to it to disable interactivity by apt-get + # and allow for automation. + # + local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" + logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}" + + logmust fakeroot debian/rules "binary-${platform}" ${debian_rules_args} + + logmust cd "$WORKDIR" + logmust mv ./*deb "artifacts/" +} + +# +# Syncing our kernel with the right upstream Canonical repo is not as +# straighforward as the other packages in linux-pkg. +# +# The Ubuntu developers maintain the timeline of the mainline kernel +# (kernel.org) in their git history. When it is time to use a new +# mainline version, they fork a new branch and then cherry-pick all their +# Ubuntu-specific generic patches on top of that and create their base +# tag. Then on top of that they cherry-pick their platform-specific +# patches (e.g. azure, aws, etc..) and create separate tags for each +# platform. This whole process is repeated for every bump in the kernel +# version (both mainline and ubuntu-specific). +# +# We want to track and sync our changes every time Canonical bumps +# their kernel version for the kernels that are used by the Delphix +# Engine that we release in order to stay up to date and lower our +# maintainance burden. As a result we do the following: +# +# * We have one kernel repo per platform. +# * Each of this repos is an Ubuntu kernel repo with our specific +# patches on top. +# * The vanilla Ubuntu kernel and our patches are divided by a single +# placeholder commit with the description "@@DELPHIX_PATCHSET_START@@". +# * Whenever the Ubuntu kernel version is bumped, we detect that +# change and use the new Ubuntu version as our base and cherry-pick +# the placeholder commit followed by our patches on top of it. +# +function kernel_update_upstream() { + local platform="$1" + + check_env UPSTREAM_GIT_URL + logmust cd "$WORKDIR/repo" + + # + # checkout our local branch that tracks upstream. + # + logmust git checkout -q upstream-HEAD + + # + # declare third-party upstream repository. + # + logmust git remote add upstream "$UPSTREAM_GIT_URL" + + # + # We get the kernel version and the ABI number from + # $_RET that is set from `get_kernel_from_platform()`. + # Example: + # + # $_RET -> 5.3.0-53-generic + # + # `cut -d '-' -f 1` of that -> 5.3.0 + # `cut -d '-' -f 3` of that -> 53 + # + # We need the kernel version and ABI number to figure + # out the latest upstream tag to sync with. + # + local kernel_version abinum + logmust get_kernel_for_platform "${platform}" + kernel_version=$(echo "$_RET" | cut -d '-' -f 1) + abinum=$(echo "$_RET" | cut -d '-' -f 2) + + # + # For each supported platform we will try to find the + # latest upstream tag to sync based on the kernel + # version and the ABI num that we got above. + # + # Note that "generic" (used mainly ESX) is a special + # case as we are currently using the HWE kernel image. + # + local tag_prefix + if [[ "${platform}" == generic ]] && + [[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then + tag_prefix="Ubuntu-hwe-${kernel_version}-${abinum}" + elif [[ "${platform}" == aws ]] || + [[ "${platform}" == azure ]] || + [[ "${platform}" == gcp ]] || + [[ "${platform}" == oracle ]]; then + + local kvers_major kvers_minor short_kvers + kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1) + kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2) + short_kvers="${kvers_major}.${kvers_minor}" + + tag_prefix="Ubuntu-${platform}-${short_kvers}-${kernel_version}-${abinum}" + else + die "assertion: unexpected platform: ${platform}" + fi + echo "note: upstream tag prefix used: ${tag_prefix}" + + # + # Query for upstream tag info based on the prefix that we've + # assembled. + # + # = Why the `tail -n 1` part? + # + # Using `git ls-remote` and `grep` with the tag's prefix alone + # may sometimes return two (and theoretically more?) results + # due to Ubuntu's "point releases". Point releases are specific + # to LTS releases and more info about them can be found in the + # links below: + # [1] https://wiki.ubuntu.com/LTS + # [2] https://wiki.ubuntu.com/PointReleaseProcess + # + # Example: + # ``` + # $ git ls-remote --tags upstream | grep Ubuntu-oracle-5.3-5.3.0-1015 + # df8fd7d8802d59 refs/tags/Ubuntu-oracle-5.3-5.3.0-1015.16_18.04.1 + # 0fe5cd29e90a5e refs/tags/Ubuntu-oracle-5.3-5.3.0-1015.16_18.04.2 + # ``` + # + # We most probably want the latest point release of a specific + # kernel thus we add `tail -n 1` in the pipeline below. + # + local upstream_tag_info + upstream_tag_info=$(git ls-remote --tags --ref upstream | grep "${tag_prefix}" | tail -n 1) + [[ -z "${upstream_tag_info}" ]] && die "could not find upstream tag for tag prefix: ${tag_prefix}" + + local upstream_tag + upstream_tag=$(echo "${upstream_tag_info}" | awk -F / '{print $3}') + [[ -z "${upstream_tag}" ]] && die "could not extract upstream tag name from the tag info" + echo "note: upstream tag: ${upstream_tag}" + logmust git fetch upstream "${upstream_tag}" + + # + # Ensure that there is a commit marking the start of + # the Delphix set of patches. Then get the hash of + # the commit right before it. + # + local dlpx_patch_end dlpx_patch_start current_ubuntu_commit + dlpx_patch_start=$(git log --pretty=oneline repo-HEAD | grep @@DELPHIX_PATCHSET_START@@ | awk '{ print $1 }') + [[ -z "${dlpx_patch_start}" ]] && die "could not find DELPHIX_PATCHSET_START" + [[ $(wc -l <<<"${dlpx_patch_start}") != 1 ]] && die "multiple DELPHIX_PATCHSET_START commits - ${dlpx_patch_start}" + current_ubuntu_commit=$(git rev-parse "${dlpx_patch_start}"^) + [[ -z "${current_ubuntu_commit}" ]] && die "could not find commit before DELPHIX_PATCHSET_START" + dlpx_patch_end=$(git rev-parse repo-HEAD) + [[ -z "${dlpx_patch_end}" ]] && die "could not find repo-HEAD's head commit" + + # + # Compare that commit with the head commit of the + # upstream tag. If the commits are the same then + # there is nothing for us to do as we are using + # the most up-to-date tag as the base for our set + # of patches. On the other hand, if the commits + # differ then it means that the upstream has been + # updated, at which point we need to cherry-pick + # our patches on top of the new upstream. + # + local upstream_head_commit + upstream_head_commit=$(git rev-parse upstream-HEAD) + [[ -z "${upstream_head_commit}" ]] && die "could not find upstream-HEAD's head commit" + + if [[ "${current_ubuntu_commit}" == "${upstream_head_commit}" ]]; then + echo "NOTE: upstream for $PACKAGE is already up-to-date." + else + # shellcheck disable=SC2086 + logmust git cherry-pick ${dlpx_patch_start}^..${dlpx_patch_end} + + logmust touch "$WORKDIR/upstream-updated" + fi + + logmust cd "$WORKDIR" +} + function post_build_checks() { # This function checks for SKIP_COPYRIGHTS_CHECK flag diff --git a/packages/linux-kernel-aws/config.sh b/packages/linux-kernel-aws/config.sh new file mode 100644 index 0000000..b42417b --- /dev/null +++ b/packages/linux-kernel-aws/config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-aws.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-aws/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + kernel_build "aws" +} + +function update_upstream() { + kernel_update_upstream "aws" +} diff --git a/packages/linux-kernel-azure/config.sh b/packages/linux-kernel-azure/config.sh new file mode 100644 index 0000000..ddb8c34 --- /dev/null +++ b/packages/linux-kernel-azure/config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-azure.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-azure/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + kernel_build "azure" +} + +function update_upstream() { + kernel_update_upstream "azure" +} diff --git a/packages/linux-kernel-gcp/config.sh b/packages/linux-kernel-gcp/config.sh new file mode 100644 index 0000000..2ba303b --- /dev/null +++ b/packages/linux-kernel-gcp/config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-gcp.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-gcp/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + kernel_build "gcp" +} + +function update_upstream() { + kernel_update_upstream "gcp" +} diff --git a/packages/linux-kernel-generic/config.sh b/packages/linux-kernel-generic/config.sh new file mode 100644 index 0000000..baa0708 --- /dev/null +++ b/packages/linux-kernel-generic/config.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-generic.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + # + # flavours=generic + # By default the generic kernel variant from Canonical + # builds both the generic and the low-latency kernel. + # We don't care about the latter. + # + kernel_build "generic" "flavours=generic" +} + +function update_upstream() { + kernel_update_upstream "generic" +} diff --git a/packages/linux-kernel-oracle/config.sh b/packages/linux-kernel-oracle/config.sh new file mode 100644 index 0000000..85b1a10 --- /dev/null +++ b/packages/linux-kernel-oracle/config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-oracle.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oracle/+git/bionic" +UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" + +function prepare() { + kernel_prepare +} + +function build() { + kernel_build "oracle" +} + +function update_upstream() { + kernel_update_upstream "oracle" +} From 185e28fe91f4199cfe277f38f82a19c198ed7a43 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 3 Sep 2020 12:24:54 -0400 Subject: [PATCH 34/39] TOOL-10140 linux-pkg: add new build lists and new package dependencies field (#117) --- .github/scripts/verify-query-packages.sh | 31 ++++++++++++++-- lib/common.sh | 47 ++++++++++++++++++++++++ package-lists/build/kernel-modules.pkgs | 8 ++++ package-lists/build/main.pkgs | 29 +++++++++++++++ package-lists/update/main.pkgs | 27 ++++++++++++++ packages/adoptopenjdk/config.sh | 2 + packages/bpftrace/config.sh | 1 + packages/connstat/config.sh | 1 + packages/delphix-kernel/config.sh | 1 + packages/delphix-sso-app/config.sh | 1 + packages/drgn/config.sh | 1 + packages/grub2/config.sh | 1 + packages/recovery-environment/config.sh | 1 + packages/targetcli-fb/config.sh | 1 + packages/zfs/config.sh | 1 + query-packages.sh | 39 +++++++++++++++++--- 16 files changed, 184 insertions(+), 8 deletions(-) create mode 100644 package-lists/build/kernel-modules.pkgs create mode 100644 package-lists/build/main.pkgs create mode 100644 package-lists/update/main.pkgs diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index c795a16..e66171f 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -30,12 +30,21 @@ read -r -a fields <<<"$(./query-packages.sh single -o git-url zfs 2>&1)" test ${#fields[@]} -eq 1 test "${fields[0]}" == 'https://github.com/delphix/zfs.git' +# Expect: "bpftrace bcc true https://github.com/delphix/bpftrace.git" +read -r -a fields <<<"$(./query-packages.sh single -o name,dependencies,can-update,git-url bpftrace 2>&1)" +test ${#fields[@]} -eq 4 +test "${fields[0]}" == 'bpftrace' +test "${fields[1]}" == 'bcc' +test "${fields[2]}" == 'true' +test "${fields[3]}" == 'https://github.com/delphix/bpftrace.git' + # Expect that "list all" outputs all directory names under packages/ diff <(ls -1 packages | sort) <(./query-packages.sh list all 2>&1 | sort) -# Expect that outputing git-url for all packages works and that the output +# Expect that outputing dependencies & git-url for all packages works and that the output # length corresponds to the number of packages. -test "$(ls -1 packages | wc -l)" -eq "$(./query-packages.sh list -o name,git-url all 2>&1 | wc -l)" +test "$(ls -1 packages | wc -l)" -eq \ + "$(./query-packages.sh list -o name,dependencies,can-update,git-url all 2>&1 | wc -l)" # Check that all package lists under package-lists\ can be loaded and that each # line of the output of the command actually refers to a package. @@ -47,9 +56,25 @@ find package-lists -name '*.pkgs' | while read -r list; do ) >/dev/null done -# Check that querying "appliance" list works +# Check that querying the built-in "appliance" list works ./query-packages.sh list appliance >/dev/null +# Check that querying lists used by the Delphix build works. +./query-packages.sh list build/main.pkgs >/dev/null +./query-packages.sh list build/kernel-modules.pkgs >/dev/null +./query-packages.sh list linux-kernel >/dev/null +./query-packages.sh list update/main.pkgs >/dev/null + +# Check that overriding TARGET_KERNEL_FLAVORS changes which kernel packages are +# returned. +test "$(./query-packages.sh list linux-kernel | wc -l | awk '{print $1}')" -gt 1 +test "$(TARGET_KERNEL_FLAVORS=generic ./query-packages.sh list linux-kernel)" == "linux-kernel-generic" + +# Check that when a package has multiple dependencies they are printed in the +# expected format. +test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o dependencies zfs)" == \ + "linux-kernel-generic,linux-kernel-aws" + # Check that the output from the appliance list contains zfs and # delphix-platform packages. Note, we explicitly do not use grep -q here as it # exits as soon as a match is found and that causes a broken pipe error as diff --git a/lib/common.sh b/lib/common.sh index 318fab4..8cc6658 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -21,6 +21,7 @@ export DEBIAN_FRONTEND=noninteractive # TODO: allow updating upstream for other branches than master export REPO_UPSTREAM_BRANCH="upstreams/master" +export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle" export UBUNTU_DISTRIBUTION="bionic" @@ -221,12 +222,14 @@ function reset_package_config_variables() { DEFAULT_PACKAGE_GIT_BRANCH DEFAULT_PACKAGE_GIT_VERSION DEFAULT_PACKAGE_GIT_REVISION + PACKAGE_DEPENDENCIES UPSTREAM_SOURCE_PACKAGE UPSTREAM_GIT_URL UPSTREAM_GIT_BRANCH WORKDIR PKGDIR PACKAGE_PREFIX + SKIP_COPYRIGHTS_CHECK " for var in $vars; do @@ -300,6 +303,24 @@ function load_package_config() { die "$PACKAGE: DEFAULT_PACKAGE_GIT_URL must begin with " \ "https:// or be set to 'none'" + local dependency + local deps_array=() + for dependency in $PACKAGE_DEPENDENCIES; do + # + # Check for special value @linux-kernel which resolves to + # all flavors of linux kernel packages. + # + if [[ $dependency == '@linux-kernel' ]]; then + logmust list_linux_kernel_packages + deps_array+=("${_RET_LIST[@]}") + continue + fi + (check_package_exists "$dependency") || + die "Invalid package dependency '$dependency'" + deps_array+=("$dependency") + done + PACKAGE_DEPENDENCIES="${deps_array[*]}" + # # Check for variables related to update_upstream() hook # @@ -501,6 +522,32 @@ function get_package_list_file() { fi } +# +# List all target kernel packages. By default, it returns all the kernel +# flavors supported and built by linux-pkg, however this can be overridden +# via TARGET_KERNEL_FLAVORS, which can be useful when testing changes against +# a specific, or even mainline kernel. +# +function list_linux_kernel_packages() { + local kernel + + _RET_LIST=() + if [[ -n "$TARGET_KERNEL_FLAVORS" ]]; then + for kernel in $TARGET_KERNEL_FLAVORS; do + (check_package_exists "linux-kernel-$kernel") || + die "Invalid entry '$kernel' in TARGET_KERNEL_FLAVORS" + _RET_LIST+=("linux-kernel-$kernel") + done + else + for kernel in $SUPPORTED_KERNEL_FLAVORS; do + check_package_exists "linux-kernel-$kernel" + _RET_LIST+=("linux-kernel-$kernel") + done + fi + + return 0 +} + function install_shfmt() { if [[ ! -f /usr/local/bin/shfmt ]]; then logmust sudo wget -nv -O /usr/local/bin/shfmt \ diff --git a/package-lists/build/kernel-modules.pkgs b/package-lists/build/kernel-modules.pkgs new file mode 100644 index 0000000..56b0eb3 --- /dev/null +++ b/package-lists/build/kernel-modules.pkgs @@ -0,0 +1,8 @@ +# +# List of kernel packages to be included in the Delphix Appliance. +# This list should not include the linux kernel packages themselves. +# + +connstat +delphix-kernel +zfs diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs new file mode 100644 index 0000000..28bffca --- /dev/null +++ b/package-lists/build/main.pkgs @@ -0,0 +1,29 @@ +# +# List of non-kernel packages to be included in the Delphix Appliance. +# + +adoptopenjdk +bcc +bpftrace +challenge-response +cloud-init +crash +crash-python +crypt-blowfish +delphix-platform +delphix-sso-app +drgn +gdb-python +grub2 +libkdumpfile +make-jpkg +makedumpfile +nfs-utils +performance-diagnostics +ptools +python-rtslib-fb +recovery-environment +savedump +sdb +targetcli-fb +td-agent-prebuilt diff --git a/package-lists/update/main.pkgs b/package-lists/update/main.pkgs new file mode 100644 index 0000000..73ff7c3 --- /dev/null +++ b/package-lists/update/main.pkgs @@ -0,0 +1,27 @@ +# +# All those packages will be auto-updated by the linux-pkg-update/userland job. +# Note that if you want to keep a package's upstream branch to be updated +# but prevent it from being merged, add the package to +# auto-merge-blacklist.pkg. +# + +# Note: we do not auto-update bcc for now as upstream tends to introduce +# changes that break bpftrace. Instead, we manually update to tagged versions. +# In the future we may want to automatically determine the latest tag and +# auto-update to that. +#bcc + +bpftrace +cloud-init +crash +drgn +grub2 +libkdumpfile +makedumpfile +nfs-utils +python-rtslib-fb +targetcli-fb + +# Do not attempt to auto-update td-agent for now as breaking changes were +# introduced upstream. +#td-agent diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh index df35453..84eab59 100755 --- a/packages/adoptopenjdk/config.sh +++ b/packages/adoptopenjdk/config.sh @@ -17,6 +17,8 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +PACKAGE_DEPENDENCIES="make-jpkg" + tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz" jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" diff --git a/packages/bpftrace/config.sh b/packages/bpftrace/config.sh index 95a1766..ef2f636 100644 --- a/packages/bpftrace/config.sh +++ b/packages/bpftrace/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bpftrace.git" DEFAULT_PACKAGE_VERSION=1.0.0 +PACKAGE_DEPENDENCIES="bcc" UPSTREAM_GIT_URL="https://github.com/iovisor/bpftrace.git" UPSTREAM_GIT_BRANCH="master" diff --git a/packages/connstat/config.sh b/packages/connstat/config.sh index 6fbb86a..8bb5e3a 100644 --- a/packages/connstat/config.sh +++ b/packages/connstat/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/connstat.git" DEFAULT_PACKAGE_VERSION="1.0.0" +PACKAGE_DEPENDENCIES="@linux-kernel" function prepare() { logmust install_pkgs \ diff --git a/packages/delphix-kernel/config.sh b/packages/delphix-kernel/config.sh index 2014156..f464dfd 100644 --- a/packages/delphix-kernel/config.sh +++ b/packages/delphix-kernel/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-kernel.git" DEFAULT_PACKAGE_VERSION="1.0.0" +PACKAGE_DEPENDENCIES="@linux-kernel" function prepare() { logmust install_pkgs \ diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 8c47e86..956fdb6 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/saml-app.git" JDK_PATH_FILE="$TOP/packages/adoptopenjdk/tmp/artifacts/JDK_PATH" +PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { java_package_exists=$(dpkg-query --show adoptopenjdk-java8-jdk >/dev/null 2>&1) diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index c5b94f6..737263a 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -18,6 +18,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/drgn.git" DEFAULT_PACKAGE_VERSION=1.0.0 +PACKAGE_DEPENDENCIES="libkdumpfile" UPSTREAM_GIT_URL="https://github.com/osandov/drgn.git" UPSTREAM_GIT_BRANCH="master" diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh index 4394e90..7d79e1b 100644 --- a/packages/grub2/config.sh +++ b/packages/grub2/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/grub2" +PACKAGE_DEPENDENCIES="zfs" UPSTREAM_GIT_URL=https://git.launchpad.net/ubuntu/+source/grub2 UPSTREAM_GIT_BRANCH=applied/ubuntu/bionic-updates diff --git a/packages/recovery-environment/config.sh b/packages/recovery-environment/config.sh index b5b0bf5..1ca3394 100644 --- a/packages/recovery-environment/config.sh +++ b/packages/recovery-environment/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/recovery-environment.git" DEFAULT_PACKAGE_VERSION=1.0.0 +PACKAGE_DEPENDENCIES="zfs" ZFS_DEB_PATH="$TOP/packages/zfs/tmp/artifacts" function prepare() { diff --git a/packages/targetcli-fb/config.sh b/packages/targetcli-fb/config.sh index 68f2ef7..7b6f406 100644 --- a/packages/targetcli-fb/config.sh +++ b/packages/targetcli-fb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/targetcli-fb.git" +PACKAGE_DEPENDENCIES="python-rtslib-fb" # Note: we get the package version programatically in build() UPSTREAM_SOURCE_PACKAGE=targetcli-fb diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index 2d02fc2..6d3f695 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" DEFAULT_PACKAGE_VERSION="0.8.0" +PACKAGE_DEPENDENCIES="@linux-kernel" UPSTREAM_GIT_URL="https://github.com/zfsonlinux/zfs.git" UPSTREAM_GIT_BRANCH="master" diff --git a/query-packages.sh b/query-packages.sh index 14fe386..b72f81a 100755 --- a/query-packages.sh +++ b/query-packages.sh @@ -19,7 +19,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 TOP="$PWD" source "$TOP/lib/common.sh" -ALL_OUTPUT_FIELDS=(name git-url dependencies) +ALL_OUTPUT_FIELDS=(name git-url dependencies can-update) function usage() { local output_fields="${ALL_OUTPUT_FIELDS[*]}" @@ -38,6 +38,7 @@ function usage() { echo " special values to the list command:" echo " all: displays all known packages" echo " appliance: displays all packages used by appliance" + echo " linux-kernel: displays all linux kernel packages" echo "" echo " -o Comma delimited output fields." echo " Possible values: ${output_fields// /, }." @@ -50,19 +51,43 @@ function usage() { function print_package() { local pkgname="$1" local outarray=() + local deps_array=() + # + # Run this in a sub-shell so that functions and variables sourced from + # load_package_config() are reset each time we load a new package. + # ( local field + local dep load_package_config "$pkgname" >/dev/null for field in "${ACTIVE_OUTPUT_FIELDS[@]}"; do case "$field" in name) outarray+=("$pkgname") ;; git-url) outarray+=("${DEFAULT_PACKAGE_GIT_URL:-none}") ;; - dependencies) outarray+=(none) ;; + dependencies) + for dep in $PACKAGE_DEPENDENCIES; do + check_package_exists "$dep" + deps_array+=("$dep") + done + if [[ ${#deps_array[@]} -eq 0 ]]; then + outarray+=(none) + else + # comma-separated list of dependencies + outarray+=("$( + IFS=, + echo "${deps_array[*]}" + )") + fi + ;; + can-update) + type -t "update_upstream" >/dev/null && + outarray+=(true) || outarray+=(false) + ;; esac done IFS=$'\t' echo "${outarray[*]}" - ) + ) || die "Failed to print info for package '$pkgname'" } function query_list() { @@ -77,14 +102,18 @@ function query_list() { list_all_packages >/dev/null elif [[ $list == appliance ]]; then # concatenate kernel and userland packages - read_package_list "$TOP/package-lists/build/kernel.pkgs" >/dev/null + read_package_list "$TOP/package-lists/build/kernel-modules.pkgs" >/dev/null local kernel_list=("${_RET_LIST[@]}") - read_package_list "$TOP/package-lists/build/userland.pkgs" >/dev/null + list_linux_kernel_packages >/dev/null + kernel_list+=("${_RET_LIST[@]}") + read_package_list "$TOP/package-lists/build/main.pkgs" >/dev/null _RET_LIST+=("${kernel_list[@]}") # check that there are no duplicates dups=$(printf '%s\n' "${_RET_LIST[@]}" | sort | uniq -d) [[ -z $dups ]] || die "Some apliance packages appear in both" \ "build/kernel.pkgs and build/userland.pkgs:\\n${dups}" + elif [[ $list == linux-kernel ]]; then + list_linux_kernel_packages >/dev/null else read_package_list "$TOP/package-lists/${list}" >/dev/null fi From 0e8793208e05c71fe300448b8c85592160f3fe9f Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 14 Oct 2020 10:19:56 -0400 Subject: [PATCH 35/39] TOOL-10145 linux-pkg rework: main linux-pkg change (#123) --- Makefile | 8 +- README.md | 577 +++++++++--------- build-info-pkg/.gitignore | 8 - build-info-pkg/build-package.sh | 89 --- build-info-pkg/debian/compat | 1 - build-info-pkg/debian/copyright | 19 - build-info-pkg/debian/install | 1 - buildlist.sh | 87 --- buildpkg.sh | 157 +---- checkupdates.sh | 80 +++ default-package-config.sh | 122 +++- jenkins-buildlist.sh | 151 ----- jenkins-updatelist.sh | 51 -- lib/common.sh | 488 ++++++++++++--- package-lists/auto-merge-blacklist.pkgs | 5 - package-lists/build/adoptopenjdk.pkgs | 7 - package-lists/build/kernel.pkgs | 14 - package-lists/build/td-agent.pkgs | 8 - package-lists/build/userland.pkgs | 33 - package-lists/update/userland.pkgs | 23 - package-lists/update/zfs.pkgs | 6 - packages/adoptopenjdk/config.sh | 16 +- packages/bcc/config.sh | 5 +- packages/bpftrace/config.sh | 8 +- packages/cloud-init/config.sh | 2 +- packages/connstat/config.sh | 2 +- packages/crash-python/config.sh | 2 +- packages/crypt-blowfish/config.sh | 2 +- packages/delphix-kernel/config.sh | 2 +- packages/delphix-platform/config.sh | 2 +- packages/delphix-sso-app/config.sh | 12 +- packages/drgn/config.sh | 8 +- packages/gdb-python/config.sh | 2 +- packages/grub2/config.sh | 9 +- packages/java8/config.sh | 9 +- packages/libkdumpfile/config.sh | 5 +- .../linux-kernel-aws/config.archive.sh | 21 +- packages/linux-kernel-aws/config.delphix.sh | 44 ++ .../linux-kernel-aws/config.prebuilt.sh | 23 +- packages/linux-kernel-aws/config.sh | 31 +- .../linux-kernel-azure/config.archive.sh | 23 +- packages/linux-kernel-azure/config.delphix.sh | 44 ++ .../linux-kernel-azure/config.prebuilt.sh | 29 + packages/linux-kernel-azure/config.sh | 31 +- packages/linux-kernel-gcp/config.archive.sh | 28 + packages/linux-kernel-gcp/config.delphix.sh | 44 ++ packages/linux-kernel-gcp/config.prebuilt.sh | 29 + packages/linux-kernel-gcp/config.sh | 31 +- .../linux-kernel-generic/config.archive.sh | 28 + .../linux-kernel-generic/config.delphix.sh | 44 ++ .../linux-kernel-generic/config.prebuilt.sh | 29 + packages/linux-kernel-generic/config.sh | 37 +- .../linux-kernel-oracle/config.archive.sh | 28 + .../linux-kernel-oracle/config.delphix.sh | 44 ++ .../linux-kernel-oracle/config.prebuilt.sh | 29 + packages/linux-kernel-oracle/config.sh | 31 +- packages/make-jpkg/config.sh | 7 +- packages/makedumpfile/config.sh | 2 +- packages/ptools/config.sh | 2 +- packages/recovery-environment/config.sh | 8 +- packages/targetcli-fb/config.sh | 3 +- packages/td-agent-prebuilt/config.sh | 3 +- packages/zfs/config.sh | 5 +- push-merge.sh | 96 +++ push-updates.sh | 139 ----- resources/delphix-secondary-mirror.key | Bin 0 -> 2255 bytes setup.sh | 47 +- sync-with-upstream.sh | 108 ++++ template/config.sh | 6 +- updatelist.sh | 282 --------- 70 files changed, 1666 insertions(+), 1711 deletions(-) delete mode 100644 build-info-pkg/.gitignore delete mode 100755 build-info-pkg/build-package.sh delete mode 100644 build-info-pkg/debian/compat delete mode 100644 build-info-pkg/debian/copyright delete mode 100644 build-info-pkg/debian/install delete mode 100755 buildlist.sh create mode 100755 checkupdates.sh delete mode 100755 jenkins-buildlist.sh delete mode 100755 jenkins-updatelist.sh delete mode 100644 package-lists/auto-merge-blacklist.pkgs delete mode 100644 package-lists/build/adoptopenjdk.pkgs delete mode 100644 package-lists/build/kernel.pkgs delete mode 100644 package-lists/build/td-agent.pkgs delete mode 100644 package-lists/build/userland.pkgs delete mode 100644 package-lists/update/userland.pkgs delete mode 100644 package-lists/update/zfs.pkgs rename build-info-pkg/debian/rules => packages/linux-kernel-aws/config.archive.sh (70%) mode change 100755 => 100644 create mode 100644 packages/linux-kernel-aws/config.delphix.sh rename build-info-pkg/debian/control.in => packages/linux-kernel-aws/config.prebuilt.sh (64%) rename build-info-pkg/clean.sh => packages/linux-kernel-azure/config.archive.sh (67%) mode change 100755 => 100644 create mode 100644 packages/linux-kernel-azure/config.delphix.sh create mode 100644 packages/linux-kernel-azure/config.prebuilt.sh create mode 100644 packages/linux-kernel-gcp/config.archive.sh create mode 100644 packages/linux-kernel-gcp/config.delphix.sh create mode 100644 packages/linux-kernel-gcp/config.prebuilt.sh create mode 100644 packages/linux-kernel-generic/config.archive.sh create mode 100644 packages/linux-kernel-generic/config.delphix.sh create mode 100644 packages/linux-kernel-generic/config.prebuilt.sh create mode 100644 packages/linux-kernel-oracle/config.archive.sh create mode 100644 packages/linux-kernel-oracle/config.delphix.sh create mode 100644 packages/linux-kernel-oracle/config.prebuilt.sh create mode 100755 push-merge.sh delete mode 100755 push-updates.sh create mode 100644 resources/delphix-secondary-mirror.key create mode 100755 sync-with-upstream.sh delete mode 100755 updatelist.sh diff --git a/Makefile b/Makefile index ad3f095..f92dd17 100644 --- a/Makefile +++ b/Makefile @@ -24,12 +24,7 @@ default: @echo 'This Makefile is only used for cleaning the repository and' @echo 'running the style checks. To build packages, first run' - @echo './setup.sh, then run ./buildlist.sh by passing the' - @echo 'appropriate package list. For instance to build the "userland"' - @echo 'package list, run:' - @echo ' ./setup.sh && ./buildlist.sh userland' - @echo 'You can also build a single package with:' - @echo ' ./setup.sh && ./buildpkg.sh ' + @echo './setup.sh, then run ./buildpkg.sh . @echo 'Refer to the README for more info.' clean: @@ -37,7 +32,6 @@ clean: @rm -rf artifacts @rm -f *.buildinfo *.changes *.deb @rm -rf update-status - @build-info-pkg/clean.sh shellcheck: shellcheck --exclude=SC1090,SC1091 \ diff --git a/README.md b/README.md index 3bd33d7..ec98f80 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,14 @@ projects. 1. [Environment Variables](#environment-variables) 1. [Package Definition](#package-definition) * [Package Variables](#package-variables) - * [Package Hooks](#package-hooks) + * [Package Hooks](#package-stages-and-hooks) * [Package Environment Variables](#package-environment-variables) * [Package WORKDIR](#package-workdir) 1. [Adding New Packages](#adding-new-packages) * [Third-party package](#third-party-package) * [In-house package](#in-house-package) 1. [Testing your changes](#testing-your-changes) +1. [Package Lists](#package-lists) 1. [Versions and Branches](#versions-and-branches) 1. [Contributing](#contributing) 1. [Statement of Support](#statement-of-support) @@ -64,7 +65,7 @@ cd linux-pkg ./setup.sh ``` -### Step 3. Build a package or a package list +### Step 3. Build a package We can now build an arbitrary package. Any package in the [packages directory](./packages) would do. Let's pick `cloud-init` as an @@ -74,34 +75,22 @@ example: ./buildpkg.sh cloud-init ``` -Packages will be stored in directory `packages/cloud-init/tmp/artifacts/`. - -Linux-pkg also allows you to build a pre-defined list of packages. Package -lists can be found in the [package-lists directory](./package-lists/build/). -Let's try to build the "userland" package list: - -``` -./buildlist.sh userland -``` - -Note that the build of the "userland" list can take close to an hour. The -artifacts for all the packages built will be located in directory `artifacts/`. +Build artifacts will be stored in directory +`packages/cloud-init/tmp/artifacts/`. ## Project Summary -There are two main tasks that are performed by this framework: building lists -of packages so that they can be later included in -[appliance-build](https://github.com/delphix/appliance-build), and keeping each -package up-to-date with its upstream project by updating the appropriate git -branches. +There are two main tasks that are performed by this framework: building +packages and keeping each package up-to-date with its upstream project by +updating the appropriate git branches. ### Building packages -This task is relatively straight forward. Every package listed in the target -package list is built and a metapackage is created with the build info of each -package that was built. -You can see section [Scripts > buildlist.sh](#buildlistsh) below for more -details. +This task is relatively straight forward. What linux-pkg calls a "package" is +really a project (usually a git project) that has a build recipe and that +produces one or more debian packages and some other metadata files. + +See [Scripts > updatelist.sh](#buildpkgsh) below. ### Updating third-party packages @@ -128,34 +117,42 @@ of the package tuned to work with our Ubuntu distribution. When updating a package, we first check if the **upstreams/master** branch is up-to-date, by fetching the latest version of the upstream git repository or the -Ubuntu source package. If changes are detected, we update **upstreams/master**. +Ubuntu source package. If changes are detected, we update the +**upstreams/master** branch and push the changes to GitHub. The second step is to check if the **master** branch is up-to-date with **upstreams/master**. If it is already up-to-date, then we are done. If not, then we attempt merging **upstreams/master** into **master**. -If the merge is successful, then we attempt building the **master** branch. The -merge is considered failed if the build fails, which means that the **master** -branch of the main repository will not be updated. +If the merge is successful, then we push the changes to a staging branch on +GitHub, called **projects/auto-update/master/merging**. The intent is for +a different system to fetch those changes, build them, and then launch tests. -Note that any updates are pushed independently to the **upstreams/master** and -**master** branches of the Delphix repository for the package. +See [Scripts > sync-with-upstream.sh](#sync-with-upstreamsh) below. -Although for now we only support auto-updating the **master** branch, the -framework is designed so that other branches could also be auto-updated. +Once the merge has been tested, [Scripts > push-merge.sh](#push-mergesh) is +called on the original VM to push the changes to the **master** branch on +GitHub. -For additional details, you can see section -[Scripts > updatelist.sh](#updatelistsh) below. +Note that the example above targets the **master** branch, but the same +workflow could apply to other branches, like **6.0/stage**, although it is +not currently in use. ## Scripts A set of scripts were created in this repository to allow easily building and updating packages both manually and through automation (e.g. Jenkins). +### query-packages.sh + +This script can be called on most unix-based systems to query metadata on the +packages built by linux-pkg. This script does not install anything on the +system, so it can be run anywhere without any side effects. + ### setup.sh Installs dependencies for the build framework. Needs to be run once to configure -the system, before any other scripts. +the system, before any other scripts (except query-packages.sh). ### buildpkg.sh @@ -167,97 +164,71 @@ Builds a single package. Package name must match a directory under ``` The build will look at `packages//config.sh` for instructions on where -to fetch the package from and how build it. The build will be performed in +to fetch the package from and how to build it. The build will be performed in `packages//tmp/`, and build artifacts for this package will be stored in the `artifacts` sub-directory. -`buildpkg.sh` includes additional options. The most common of them is `-u`, -which will update the package with upstream. See section -[Updating third-party packages](#updating-third-party-packages) for more info. - -### buildlist.sh +Note that if the build of the package depends on build artifacts from another +linux-pkg package, those will be fetched from a predetermined S3 location. -Builds a list of packages and the assossiated metapackage. The list of packages -is read from [package-lists/build/{list}.pkgs](./package-lists/build/) and -builds the packages listed there by invoking `buildpkg.sh` on each one of them. -Once they are all built, it builds the metapackage, which stores the build info -for each package that was built -- info such as git hash, git url and git -branch. +### checkupdates.sh +Usage: ``` -./buildlist.sh +./checkupdates.sh ``` -`buildlist.sh` doesn't take build options but can be configured by passing -various environment variables. -See section [Environment Variables](#environment-variables) for more details. +This checks if a package has updates in the upstream project that haven't been +pulled into the **upstreams/master** branch, or if the **upstreams/master** +branch has commits that haven't been merged into the **master** branch. -Packages are built sequentially in the order defined in the pkgs file. If a -package fails to build, the script exits immediately. +If updates are available, the file `/update-available` will be created. -### jenkins-buildlist.sh +The intention of this script is to inform the caller whether an update job +should be called for the given package. -This is a wrapper script around `buildlist.sh` and was designed -to be called by Jenkins. Any environment variables that are passed to -`jenkins-buildlist.sh` are propagated to the child script. In addition, -`jenkins-buildlist.sh` interprets environment variables specified in section -[Environment variables specific to jenkins-buildlist](#environment-variables-specific-to-jenkins-buildlist). - -### updatelist.sh - -Updates all the packages listed in package list -[package-lists/update/{list}.pkgs](./package-lists/update/): +### sync-with-upstream.sh +Usage: ``` -./updatelist.sh [-n] +./sync-with-upstream.sh ``` -Here are the steps for updating one package: - -1. Run `buildpkg.sh -u `. This will attempt to update the - **upstreams/master** branch, and then attempt to merge **upstreams/master** - into **master**. If changes are detected on **master**, then the package will - be built. If a package is listed in - [package-lists/auto-merge-blacklist.pkgs](./package-lists/auto-merge-blacklist.pkgs), - then `-M` will be passed to `buildpkg.sh` and we will not attempt updating - **master**. - -1. If changes are detected for **upstreams/master** or **master**, push them to - the default repository for the package (e.g. `github.com/delphix/`). - This is done by invoking `push-updates.sh` for each branch. Note that - **upstreams/master** will be updated even if merge with **master** failed, - allowing developers to later perform the merge manually. +This script has 2 tasks: +1. Check if the upstream project has updates that are not pulled into the +**upstreams/master** branch of the package, and if so then update that branch +and push changes to GitHub. +2. Merge **upstreams/master** into **master** and push the changes to a staging +branch on GitHub, called **projects/auto-update/master/merging**. Another +system should use that branch to build the package, and then run the appropriate +integration tests. -Each package is processed independently, so a failure to update one package -doesn't affect the update of other packages. A report is generated at the end. +After testing has been completed, `push-merge.sh ` should be called on +the same system to push the merge to the **master** branch. -`updatelist.sh` can be run in dry-run mode by passing `-n` so that package -updates are not pushing to the target repository. Its behaviour can also be -configured by passing various environment variables. See section -[Environment Variables](#environment-variables) for more details. +Note that the DRYRUN environment variable must be set when running this script. +If DRYRUN is set to "true", then changes are not pushed to GitHub in step 1, +and staged changes are pushed to **projects/auto-update/master/merging-dryrun** +in step 2 instead of the non-dryrun branch. The intention is that when testing +changes to the logic we want to be able to run most of the logic, but without +affecting the production branches. -### jenkins-updatelist.sh - -This is a wrapper script around `updatelist.sh` and was designed to be called -by Jenkins. Any environment variables that are passed to `jenkins-buildlist.sh` -are propagated to the child script. - -### push-updates.sh - -This script pushes branch updates to the default repository for the package. It -should be called after running `buildpkg.sh -u `. The script should be -invoked with: +### push-merge.sh +Usage: ``` -./push-update.sh -u|-m +./push-merge.sh ``` -Running it with `-u` will update **upstreams/master** and running it with `-m` -will update **master**. +This must be called on a system that has previously called +`sync-with-upstream.sh` for the same package. It will push the merge that was +previously prepared by `sync-with-upstream.sh` to the production **master** +branch, after checking that the **master** branch hasn't been modified since +`sync-with-upstream.sh` was called. -Note that credentials for a user that has permissions to push to the target -repository must be passed. Passing the `-n` option does a dry-run, meaning that -the target repository won't be updated (`-n` will be passed to `git push`). +Like for `sync-with-upstream.sh`, the DRYRUN environment variable must be set +to run this script. However, the script will fail unless DRYRUN is set to +"false" given that there is not much that can be tested in dry-run mode. ## Environment Variables @@ -268,61 +239,75 @@ of some of the scripts defined above. we are running on the appropriate Ubuntu distribution in AWS. Affects all scripts. -* **DRY_RUN**: Set to "true" to prevent `updatelist.sh` from updating production - package repositories. `updatelist.sh` will invoke `push-updates.sh` with `-n`. +* **DRYRUN**: Must be set to either "true" of "false" when running script + [sync-with-upstream.sh](#sync-with-upstreamsh), and to "false" when running + script [push-merge.sh](#push-mergesh). * **PUSH_GIT_USER, PUSH_GIT_PASSWORD**: Set to the git credentials used to push - updates to package repositories. Affects `updatelist.sh` and - `push-updates.sh`. + updates to package repositories. Affects scripts + [sync-with-upstream.sh](#sync-with-upstreamsh) and + [push-merge.sh](#push-mergesh). * **DEFAULT_REVISION**: Default revision to use for packages that do not have a revision defined. If not set, it will be auto-generated from the timestamp. - Applies to `buildpkg.sh` and `buildlist.sh`. - -* **DEFAULT_BRANCH**: Default git branch to use when fetching a package that - does not have a branch explicitly defined. If not set, it will default to - "master". Applies to `buildpkg.sh` and `buildlist.sh`. - -* **CHECKSTYLE**: Applies to `buildlist.sh`. Passes `-c` to `buildpkg.sh` when - `CHECKSTYLE` is "true" to execute the `checkstyle` hook when building a package. - See [Package Definition](#package-definition) section for more details about - the hook. - -* **TARGET_PLATFORMS**: Some packages build kernel modules. This specifies which - kernel versions to build those packages for and accepts a space-separated list - of values. If the value is a platform, such as "aws" or "generic", then it - will auto-determine the default kernel version for the provided platform. If - `TARGET_PLATFORMS` is unset or "default", then it will build for all supported - platforms. - -* **UPDATE_PACKAGE_NAME**: Applies to `updatelist.sh` only. If this variable is - set then `updatelist.sh` only updates the package specified by this variable. - -* **{PACKAGE}_GIT_URL, {PACKAGE}_GIT_BRANCH, {PACKAGE}_VERSION, - {PACKAGE}_REVISION**: Can be used to override defaults for a given package. - `{PACKAGE}` is the package name in upper case with `-` converted to `_`. For + Applies to [buildpkg.sh](#buildpkgsh). + +* **DEFAULT_GIT_BRANCH**: The product branch that is being built or updated is + typically stored in the file `branch.config`, however it can be overridden via + DEFAULT_GIT_BRANCH. The product branch is used in multiple instances. When + running [setup.sh](#setupsh), it will determine what linux-package-mirror + link to use when fetching packages from apt (although those links can be + overridden via DELPHIX_PACKAGE_MIRROR_MAIN and + DELPHIX_PACKAGE_MIRROR_SECONDARY). When running [buildpkg.sh](#buildpkgsh), + it will determine which branch to fetch from the package's repository, unless + it is overridden via `-b`; if the package has build-dependencies on other + linux-pkg packages, those dependencies will be fetched from an S3 url that is + versioned based on the product branch (although the package dependencies + URLs can be overridden via package_S3_URL variables). Finally, when running + [sync-with-upstream.sh](#sync-with-upstreamsh) or + [push-merge.sh](#push-mergesh) it defines what branch of the package is + being updated. + +* **TARGET_KERNEL_FLAVOURS**: Some packages have build dependencies on the + linux kernel. Those packages have `PACKAGE_DEPENDENCIES="@linux-kernel"` in + their `config.sh`. By default, those packages are built for all the supported + kernel flavours (see SUPPORTED_KERNEL_FLAVORS in `common.sh`), however it is + possible to restrict which kernel flavours those packages are built for. + +* **package_GIT_URL, package_GIT_BRANCH, package_VERSION, + package_REVISION**: Can be used to override defaults for a given package. + `package` is the package name in upper case with `-` converted to `_`. For instance `CLOUD_INIT_GIT_BRANCH=feature1` would set the branch to fetch package `cloud-init` from to `feature1`. This is useful when running `buildlist.sh` to override defaults for multiple packages. Applies to both `buildlist.sh` and `buildpkg.sh`. -### Environment variables specific to jenkins-buildlist - -* **SINGLE_PACKAGE_NAME**: When running `jenkins-buildlist.sh`, other - `SINGLE_PACKAGE_{*}` parameters mentioned below are used to override the - defaults for this package. - -* **SINGLE_PACKAGE_GIT_URL, SINGLE_PACKAGE_GIT_BRANCH, SINGLE_PACKAGE_VERSION, - SINGLE_PACKAGE_REVISION**: Applies to `jenkins-buildlist.sh` only. Those are - equivalent to the `{PACKAGE}_{*}` variables described previously but apply to - the package passed in `SINGLE_PACKAGE_NAME`. They are added for convenience - when using Jenkins. - -* **CUSTOM_BUILDER_ENV**: Applies to `jenkins-buildlist.sh` only. This is a - multi-line field that takes one `{PACKAGE}_{*}=value` entry per line and is - parsed by `jenkins-buildlist.sh` to set the specified `{PACKAGE}_{*}` - environment variables. This can be used to set any number of `{PACKAGE}_{*}` - variables from Jenkins. +* **package_S3_URL**: Similar to the package_VAR variables above. This is used + to override the default S3 location for where package build-dependencies are + fetched for a given linux-pkg package. For instance, if you are building + bpftrace, which has `PACKAGE_DEPENDENCIES="bcc"` in its config, the + `fetch_dependencies()` stage in the build will fetch the latest build + artifacts of the bcc package from a predetermined S3 location. If you pass + `BCC_S3_URL=s3://path/to/custom/bcc/artifacts` then those artifacts will be + fetched insteasd. + +* **DELPHIX_PACKAGE_MIRROR_MAIN, DELPHIX_PACKAGE_MIRROR_SECONDARY**: When + the [setup.sh](#setupsh) script is run, it will configure the apt sources + to point to versioned delphix mirrors of the Ubuntu archive (MAIN mirror) + and of some auxiliary archives (SECONDARY mirror). Delphix has many snapshots + of those mirrors at different points in time, and if you want to use a custom + snapshot, you can pass it in those environment variables. + +* **JENKINS_OPS_DIR**: When fetching artifacts from other linux-pkg packages + that are marked as dependencies of a package, by default we look for a + specific S3 path that contains production package artifacts generated by + post-push jobs of the ops Jenkins agent. The production ops Jenkins agent + stores artifacts in the special `jenkins-ops` sub-directory. When using + a developer ops Jenkins agent, it stores build artifacts in a different S3 + sub-directory: `jenkins-ops.`. By setting JENKINS_OPS_DIR to that + sub-directory you can instruct linux-pkg to fetch artifacts of build + dependencies produced by the developer Jenkins instance instead of the + production one. ## Package Definition @@ -344,6 +329,13 @@ Here is a list of variables that can be defined for a package: `https://` URL. One exception is if the source of the package being built isn't fetched from git. In this case, set this to "none". +* **PACKAGE_DEPENDENCIES**: (Optional) If the build of this package requires + fetching artifacts from other linux-pkg packages, those should be specified + in PACKAGE_DEPENDENCIES, as a space-separated list. The dependencies will + be fetched in the `fetch_dependencies()` step into `//` where + "dep" is the dependency's name. A special value can be passed for packages + that target all the supported flavours of the linux-kernel: `@linux-kernel`. + * **DEFAULT_PACKAGE_GIT_BRANCH**: (DEPRECATED) Default git branch to use when fetching from or pushing to `DEFAULT_PACKAGE_GIT_URL`. This should be typically left unset. The branch to fetch the package from defaults @@ -355,88 +347,124 @@ Here is a list of variables that can be defined for a package: * **DEFAULT_PACKAGE_VERSION**: (Optional) The version of the package is set to this value when it is built. **Note:** If this field is not set, then you - should provide a mechanism in the [build](#build) hook to auto-determine the - version from the source code. + should provide a mechanism in the [build](#build-hook) hook to auto-determine + the version from the source code. + WARNING: This parameter will be removed in the near future, as we will rely on + the changelog contained in the package's repository to get the package version + in the future. * **DEFAULT_PACKAGE_REVISION**: (Optional) The revision of the package is set to this value when it is built (note that the full version of a package is "_VERSION-REVISION_"). If unset, it defaults to value of environment variable DEFAULT_REVISION. + WARNING: This parameter is currently unused and will be removed in the near + future. * **UPSTREAM_SOURCE_PACKAGE**: (Optional) Third-party packages that have an - [update_upstream](#update-upstream) hook and are updated from an Ubuntu source - package should set this to the name of the source package. + [update_upstream](#update-upstream-hook) hook and are updated from an Ubuntu + source package should set this to the name of the source package. * **UPSTREAM_GIT_URL, UPSTREAM_GIT_BRANCH**: (Optional) Third-party packages - that have an [update_upstream](#update-upstream) hook and are updated from a - git repository should set this to the upstream git url and branch. - -### Package hooks - -This is a list of hooks that can be defined for a package. Those are simply bash -functions that are called by `buildpkg.sh`. - -#### Prepare - -The `prepare()` hook is optional. It is called before calling the build hook and -normally installs the build dependencies for the package. - -#### Fetch + that have an [update_upstream](#update-upstream-hook) hook and are updated + from a git repository should set this to the upstream git url and branch. + +* **FORCE_PUSH_ON_UPDATE**: (Optional) This applies to some third-party packages + that have an [update_upstream](#update-upstream-hook) hook. Most third-party + packages are synced with upstream by performing a simple "git-merge" command, + so when the merge is pushed it can be done with "git push". However some + packages, like the linux-kernel ones, perform a rebase instead, and so the + merge must be force-pushed instead. If you want to use force push to push + an auto-merge, set FORCE_PUSH_ON_UPDATE to "true". Note that a safety check + is always performed prior to doing the push to make that the target branch + has not changed since the auto-merge commit was generated, however disabling + force-push by default is an extra precaution. + +* **SKIP_COPYRIGHTS_CHECK**: (Optional) By default, at the end of a package's + build we check that each produced deb contains a copyright file, unless + SKIP_COPYRIGHTS_CHECK is set to "true". + +### Package stages and hooks + +When operations are performed on a package by build or auto-update scripts, +such as [buildpkg.sh](#buildpkgsh) or +[sync-with-upstream.sh](#sync-with-upstreamsh), those operations are usually +split into high-level tasks called "stages". Some of those stages can be +modified or must be defined in a package's config file, so we refer to them +here as "hooks". Hooks that have a default definition are stored in +the `default-package-config.sh` file. + +Other "stages" are not meant to be modified and aren't functionally different +from regular function calls, we want to give them more visibility in the build +process as they are deemed as important high-levels tasks, so they are called +via the `stage()` helper function. + +#### Fetch (hook) The `fetch()` hook is optional, as a default is provided and should be used. It is called when fetching the source code of the package to build or to update. -The repository is cloned into `packages//tmp/repo` and checked out as +The repository is cloned into `/repo` and checked out as branch **repo-HEAD**. If we are performing a package update, then we also fetch the **upstreams/master** branch into **upstream-HEAD**. The default should only be overridden when not fetching the package source from git. -#### Build +#### Prepare (hook) + +The `prepare()` hook is optional. It is called before calling the build hook and +normally installs the build dependencies for the package. + +#### Build (hook) The `build()` hook is mandatory. It is responsible for building the package and storing the build products into `packages//tmp/artifacts/`. -#### Store Build Info +#### Update Upstream (hook) -The `store_build_info()` hook is optional. It is called right after the -build hook. It is responsible of creating the `/build_info` file that -contains information about the source of the code used to build the package. +The `update_upstream()` hook should only be defined for third party packages +that can be auto-updated. It is responsible for fetching the latest upstream +source code on top of branch **upstream-HEAD** of our fetched repository in +`/repo`. Note that any changes should be rebased on top of +the **upstreams/master** branch. If changes are detected, file +`/upstream-updated` should be created. -A default hook is provided in `default-package-config.sh` and will -be used if it is not overriden. If the package comes from a git repository, -the default hook will store the git hash, branch and repository of the -package's source. +#### Merge With Upstream (hook) -`build_info` files for each package are consumed by the -[metapackage](./build-info-pkg) when running [buildlist.sh](#buildlistsh). +The `merge_with_upstream()` hook is called after the `update_upstream()` hook +when a package is updated via [sync-with-upstream.sh](#sync-with-upstreamsh). +Whereas `update_upstream()` updates the **upstream-HEAD** branch, +`merge_with_upstream` then merges the **upstream-HEAD** branch into the +**repo-HEAD** branch. For most third-party packages this can be left unset as +the default will be used. For packages that have a more complex merge strategy, +such as the linux-kernel packages, this hook can be used. -#### Post Build Checks +#### Checkstyle (hook) -The `post_build_checks()` hook is optional. It is responsible for checking if -the debian package being built has copyright file associated with it in the -right location. This file will be used to generate the license information for -the appliance. +The `checkstyle()` hook is optional. It is called before building the package if +`-c` is provided to `buildpkg.sh`. Note that this hook isn't currently used by +our build automation and is more of a prototype for an idea. -A default hook is provided in `default-package-config.sh` and will -be used if it is not overriden. The packages that are eligible to skip this check -should define `SKIP_COPYRIGHTS_CHECK=true` in their respective `config.sh` files. +#### Fetch Dependencies -#### Checkstyle +`fetch_dependencies` is an immutable stage. It is called for fetching build +artifacts from other linux-pkg packages that are required for performing the +build. See the PACKAGE_DEPENDENCIES package variable for mroe info. -The `checkstyle()` hook is optional. It is called before building the package if -`-c` is provided to `buildpkg.sh`. +#### Store Build Info -#### Update Upstream +`store_build_info()` is an immutable stage. It is called after the `build()` +stage. It is responsible for storing some build info / metadata, such as the +git hash used to perform the build. Some of the build info that is stored is +used by build automation, so care must be exercised when modifying it. -The `update_upstream()` hook should only be defined for third party packages -that need to be auto-updated. It is responsible for fetching the latest upstream -source code on top of branch **upstream-HEAD** of our fetched repository in -`packages//tmp/repo`. Note that any changes should be rebased on top of -the **upstreams/master** branch. If changes are detected, file -`packages//tmp/upstream-updated` should be created. +#### Post Build Checks -After the `update_upstream()` hook is called, and if changes are detected, -`buildpkg.sh` will proceed to merge the **upstream-HEAD** branch into -**repo-HEAD** and build the resulting code. +`post_build_checks()` is an immutable stage. It is responsible for performing +post-build checks that are common to all packages. + +One of the checks verifies that each debian package produced has a copyright +file associated with it in the right location. This file is used elsewhere in +the product to generate the license information for the appliance. This check +can be skipped for a package by defining `SKIP_COPYRIGHTS_CHECK=true` in its +config file. ### Package environment variables @@ -461,7 +489,9 @@ variables are set-up by the framework. Here is a quick list: ### Package WORKDIR Each package is being fetched, built and updated in directory -`linux-pkg/packages//tmp/`, referred to as `WORKDIR`. +`linux-pkg/packages//tmp/`, referred to as `WORKDIR`. Whenever a +script is called to operate a package, the WORKDIR directory is recreated and +a `linux-pkg/workdir` symlink is created that points to this WORKDIR. The following sub-directories are created in `WORKDIR`: @@ -472,15 +502,14 @@ The following sub-directories are created in `WORKDIR`: * **source**: where the source package is fetched when updating upstream from a source package. -The following files are used as status indicators in `WORKDIR`: - -* **building**: created when package is being built, deleted on success. +The following files are created in `WORKDIR`: -* **updating-upstream**: created when updating upstream branch, deleted on - success. +* **upstream_tag**: During a package's auto-update, we may wish to also push + a tag fetched from the upstream repository for informational purposes. If so, + the `upstream_tag` file should be created and contain the name of the tag + that needs to be pushed. -* **merging**: created when package is being merged with upstream branch, - deleted on success. +The following files are used as status indicators in `WORKDIR`: * **upstream-updated**: created if **upstream-HEAD** has updates that should be pushed. @@ -488,18 +517,16 @@ The following files are used as status indicators in `WORKDIR`: * **repo-updated**: created if **repo-HEAD** has updates that should be pushed, following a merge. -* **build_info**: created by the `store_build_info()` package hook. - ## Adding new packages When considering adding a new package, the workflow will depend on whether the package is a [third-party package](#third-party-package) or [in-house package](#in-house-package). -**Note For Delphix Employees**: +**Note:**: If you are thinking of adding a new package to this framework, you should first read the -[Delphix Open-Source Policy](https://docs.delphix.com/cto/ip-strategy/outbound-open-source). +[Delphix Open-Source Policy](https://docs.delphix.com/en/ip-strategy/outbound-open-source). ### Third-party package @@ -551,14 +578,23 @@ branch that points to the **master** branch; you can then update `DEFAULT_PACKAGE_GIT_URL` in config.sh to your forked git repository and skip to step 6. -You can fetch the upstream source code by running: +You can fetch the upstream source code from an Ubuntu source package by running: ``` -./buildpkg.sh -i +cd packages//tmp/ +mkdir source +cd source +apt-get source +cd .. +mv source/""*/ repo +cd repo +git init +git checkout -b repo-HEAD +git add -f . +git commit -m '' ``` - -This will automatically fetch the code into `packages//tmp/repo` and -initialize it as a git repository. +TODO: create a command that will run the steps above. It used to be done by +`buildpkg.sh -i`, but this logic has been removed. #### Step 4. Create a developer repository @@ -573,20 +609,11 @@ e.g. DEFAULT_PACKAGE_GIT_URL="https://github.com//" ``` -Note that the branch will default to **master** unless -`DEFAULT_PACKAGE_GIT_BRANCH` is also provided. - #### Step 5. Push to your developer repository -Next step is to push the upstream code to the newly created repository using the -`push-update.sh` script. The script will need to be called twice: once for the -**upstreams/master** branch and once for the **master** branch. It will also -prompt you for your git credentials. - -``` -./push-updates -u -./push-updates -m -``` +Next step is to push the upstream code to the newly created repository to your +developer repository. You should push the initial commit to both the **master** +branch and the **upstreams/master** branch. #### Step 6. Build the package @@ -599,8 +626,10 @@ that will install those build dependencies. For an Ubuntu source package, those dependencies can be installed by calling `install_build_deps_from_control_file()`. For other packages, you can usually find the build dependencies in the project's -README. It is recommended to use the `install_pkgs()` function to install -packages. +README. It is recommended to edit the `debian/control` file of the package +to list the required build dependencies, so that +`install_build_deps_from_control_file()` can be used. Otherwise, you can also +use the `install_pkgs()` lib function to install packages. Next step is to add a [build()](#build) hook. It is recommended to use the `dpkg_buildpackage_default()` function. @@ -632,8 +661,8 @@ Once this is all ready, you can try building the package by running: #### Step 7. Make the package auto-updatable If you want the package to be automatically updated with upstream (strongly -recommended), you'll need to add the [update_upstream()](#update-upstream) hook -to `config.sh`. You should use the following functions provided by +recommended), you'll need to add the [update_upstream()](#update-upstream-hook) +hook to `config.sh`. You should use the following functions provided by [lib/common.sh](./lib/common.sh): * `update_upstream_from_source_package()` if `UPSTREAM_SOURCE_PACKAGE` is set. @@ -723,28 +752,15 @@ require a debian metadirectory. #### Add package to package-lists -* Add the new package to the appropriate build list in - [package-lists/build/](./package-lists/build/). - Most packages that will be deployed on the Delphix Appliance should be added - to the [userland.pkgs](./package-lists/build/userland.pkgs) list. - -* If this is a third-party package that is to be auto-updated by - `updatelist.sh`, it should also be added to - [package-lists/update/userland.pkgs](./package-lists/update/userland.pkgs). - -* To make sure that the new package is included in the Delphix Appliance by - appliance-build, it should be added as a dependency to an existing package - such as `delphix-platform` or `delphix-virtualization`. +See the [Package Lists](#package-lists) section for more info. #### Make the package official -**Note**: this step only applies to Delphix. - Once your new package builds and has been tested in the product, the next step is to create an official repository for it. 1. First, you should read - [Delphix Open-Source Policy](https://docs.delphix.com/cto/ip-strategy/outbound-open-source) + [Delphix Open-Source Policy](https://docs.delphix.com/en/ip-strategy/outbound-open-source) if you haven't already, and provide the necessary info so that a `github.com/delphix/` repository can be created for it. You'll need to push the **master** branch from your developer repository, as well as the @@ -768,34 +784,50 @@ package managed by linux-pkg: 1. Run `git-ab-pre-push` from your package's repository. -More instructions available -[here](https://docs.google.com/document/d/1pD0AusWAIbqXalx-B5nhrrHBfMme6wHvJG9c7O_wqb4/view). +TODO: complete section ### Testing changes to linux-pkg -If you are adding a new package, changing the linux-pkg framework, or changing -the build definition (config.sh file) of a package, you should perform the -following steps: +TODO: complete section + +## Package Lists -1. On your build VM, in the linux-pkg repository, run checkstyle: +Package lists are basically just lists of packages defined in linux-pkg. +They are mainly consumed by the Jenkins build infrastructure by calling +the [./query-packages.sh](./query-packages.sh) utility. Jenkins needs to know +which packages to build and include for a given version of the Delphix +appliance. - ``` - make clean - make check - ``` +Package lists are stored under [./package-lists](./package-lists), in two +sub-directories: `build` and `update`. The `build` directory contains packages +that are built and consumed by the Delphix Appliance, while the `update` +directory contains a list of packages that are automatically synced with +the upstream projects. -1. Run the Jenkins build jobs for the - [userland](http://selfservice.jenkins.delphix.com/job/devops-gate/job/master/job/linux-pkg-build/job/master/job/kernel/job/pre-push/) - and - [kernel](http://selfservice.jenkins.delphix.com/job/devops-gate/job/master/job/linux-pkg-build/job/master/job/userland/job/pre-push/) - build lists. +There are two physical `build` lists: -1. Run the Jenkins update job for the - [userland](http://selfservice.jenkins.delphix.com/job/devops-gate/job/master/job/linux-pkg-update/job/master/job/userland/job/update/) - update list. +* `main.pkgs`: this is the default list for packages that are to be added to the + Delphix Appliance. -More instructions available -[here](https://docs.google.com/document/d/1pD0AusWAIbqXalx-B5nhrrHBfMme6wHvJG9c7O_wqb4/view). +* `kernel-modules.pkgs`: this list is similar to the `main` list but contains + packages that have a dependency on the multiple flavours of the linux kernel + that are supported by the Delphix Appliance. + +There's also a virtual build list, called "linux-kernel", which lists all the +linux kernel packages built by linux-pkg (one for each supported flavour of +the linux kernel). You can list the contents of the virtual list by running: + +``` +./query-packages.sh list linux-kernel +``` + +There is a single `update` list called `main.pkgs`, which contains all the +packages that are auto-updated nightly by Jenkins. Note that zfs is not in +that list as it has a dedicated Jenkins job that tracks the upstream +repository and launches as soon as there are new changes. + +Most third-party packages should have an `update_upstream()` hook defined and +be added to that list. ## Versions and Branches @@ -815,7 +847,7 @@ linux-pkg repository itself follows the Delphix branching policy outlined [here](https://docs.delphix.com/pages/viewpage.action?spaceKey=RE&title=New+Branching+Mechanism). When creating a new branch or release for the Delphix Appliance, an external script should create the relevant branch or tag for each repository. The -branch or tag should then be passed to the build in the `DEFAULT_BRANCH` +branch or tag should then be passed to the build in the `DEFAULT_GIT_BRANCH` environment variable. ### Future work @@ -825,15 +857,6 @@ image will need to be picked accordingly. We are currently using `bootstrap-18-04`, but this will not be the case anymore once we switch to a newer Ubuntu distribution. -Regarding auto-update of third-party packages, we'll most likely want to enable -support for other branches than master, especially _stage_ ones. This way we'd -be able to automatically pull in security updates for our third-party packages -that track Ubuntu source packages. - -This means that we will also need integration with our Ubuntu package mirrors. -The auto-update process will need to track the proper archive when fetching -source packages. - ## Contributing All contributors are required to sign the Delphix Contributor Agreement prior diff --git a/build-info-pkg/.gitignore b/build-info-pkg/.gitignore deleted file mode 100644 index 1e596e8..0000000 --- a/build-info-pkg/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -artifacts -debian/changelog -debian/control -debian/.debhelper -debian/delphix-buildinfo* -debian/files -debian/debhelper-build-stamp -lib \ No newline at end of file diff --git a/build-info-pkg/build-package.sh b/build-info-pkg/build-package.sh deleted file mode 100755 index 2ce114f..0000000 --- a/build-info-pkg/build-package.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -e -# -# Copyright 2018, 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2016 - -# -# This script is responsible of building the build-info package after a list -# of packages have been built. Each package may generate a build_info file -# containing custom metadata related to the build (e.g. see the store_git_info -# function in common.sh). This script concatenates the build_info files -# generated by each package into a single file that will be provided by this -# package: /lib/delphix-buildinfo/{pkg_list}.info. -# -# The name of the build-info package is "delphix-buildinfo-{pkg_list}" -# - -TOP="$(git rev-parse --show-toplevel)" -source "$TOP/lib/common.sh" - -SCRIPT_DIR="$TOP/build-info-pkg" - -[[ $# -eq 1 ]] || die "Must provide package list as argument." - -pkg_list="$1" -logmust get_package_list_file "build" "$pkg_list" -pkg_list_file="$_RET" - -logmust "$SCRIPT_DIR/clean.sh" - -logmust read_package_list "$pkg_list_file" -PACKAGES=("${_RET_LIST[@]}") - -INFO_FILE="$SCRIPT_DIR/lib/delphix-buildinfo/${pkg_list}.info" -logmust mkdir -p "$(dirname "$INFO_FILE")" - -LINUX_PKG_HASH="$(git rev-parse HEAD)" || die "git rev-parse HEAD failed" - -# -# LINUX_PKG_GIT_URL & LINUX_PKG_GIT_BRANCH are passed by Jenkins -# -cat <<-EOF >"$INFO_FILE" - Linux-pkg Package Framework: - Git hash: $LINUX_PKG_HASH - Git repo: ${LINUX_PKG_GIT_URL:-unknown} - Git branch: ${LINUX_PKG_GIT_BRANCH:-unknown} -EOF - -echo "" >>"$INFO_FILE" - -logmust cd "$TOP/packages" -for pkg in "${PACKAGES[@]}"; do - echo "Package $pkg:" - if [[ -f "${pkg}/tmp/build_info" ]]; then - cat "${pkg}/tmp/build_info" - else - echo "NO INFO" - fi - echo "" -done >>"$INFO_FILE" - -package_name="delphix-buildinfo-${pkg_list}" - -logmust cd "$SCRIPT_DIR" -logmust bash -c "sed 's/@@PACKAGE@@/$package_name/g' \ - debian/control.in >debian/control" - -export DEBEMAIL="Delphix Engineering " -export PACKAGE_VERSION=1.0.0 -export PACKAGE_REVISION=${DEFAULT_REVISION:-0} - -logmust set_changelog "$package_name" -logmust dpkg-buildpackage -uc -us -b -logmust mkdir -p artifacts -logmust mv ../"$package_name"*deb artifacts/ -logmust rm -f ../"$package_name"*.buildinfo ../"$package_name"*.changes -logmust cp "$INFO_FILE" artifacts/build-info diff --git a/build-info-pkg/debian/compat b/build-info-pkg/debian/compat deleted file mode 100644 index f599e28..0000000 --- a/build-info-pkg/debian/compat +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/build-info-pkg/debian/copyright b/build-info-pkg/debian/copyright deleted file mode 100644 index 5bd3179..0000000 --- a/build-info-pkg/debian/copyright +++ /dev/null @@ -1,19 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ - -Files: * -Copyright: 2018 Delphix -License: Apache-2.0 - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - . - http://www.apache.org/licenses/LICENSE-2.0 - . - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - . - On Debian systems, the complete text of the Apache License, Version 2 - can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/build-info-pkg/debian/install b/build-info-pkg/debian/install deleted file mode 100644 index 502167f..0000000 --- a/build-info-pkg/debian/install +++ /dev/null @@ -1 +0,0 @@ -/lib diff --git a/buildlist.sh b/buildlist.sh deleted file mode 100755 index e0b627b..0000000 --- a/buildlist.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# -# Copyright 2018, 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script first builds a list of packages by running buildpkg.sh on each -# package, and then generates a build-info package. All the build products are -# stored in the ./artifacts directory. Valid package lists are stored in -# package-lists/build/ -# - -TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -source "$TOP/lib/common.sh" - -logmust check_running_system - -function usage() { - [[ $# != 0 ]] && echo "$(basename "$0"): $*" - echo "Usage: $(basename "$0") " - echo "" - echo " This script fetches and builds all the packages defined in" - echo " package-lists/build/.pkgs." - echo "" - exit 2 -} - -[[ $# -eq 1 ]] || usage "takes exactly one argument." >&2 - -pkg_list="$1" -logmust get_package_list_file "build" "$pkg_list" -pkg_list_file="$_RET" - -logmust cd "$TOP" -logmust make clean -logmust mkdir artifacts - -# -# Auto-generate the default revision for all the packages. It will be the -# default used if the revision is not set explicitly anywhere else. -# -export DEFAULT_REVISION="${DEFAULT_REVISION:-$(default_revision)}" - -# -# A list of target platform or versions to build modules for can be passed in -# TARGET_PLATFORMS. Convert values like "aws" into actual kernel -# versions and store them into KERNEL_VERSIONS. -# -logmust determine_target_kernels -export KERNEL_VERSIONS - -build_flags="" -if [[ "$CHECKSTYLE" == "true" ]]; then - build_flags="${build_flags} -c" -fi - -# -# Get the list of packages to build. -# -logmust read_package_list "$pkg_list_file" -PACKAGES=("${_RET_LIST[@]}") - -for pkg in "${PACKAGES[@]}"; do - # shellcheck disable=SC2086 - logmust ./buildpkg.sh $build_flags "$pkg" -done - -logmust build-info-pkg/build-package.sh "$pkg_list" -logmust cp build-info-pkg/artifacts/* artifacts/ - -for pkg in "${PACKAGES[@]}"; do - logmust cp "packages/$pkg/tmp/artifacts"/* artifacts/ -done - -echo_success "Packages have been built successfully." diff --git a/buildpkg.sh b/buildpkg.sh index edd3560..04d9a54 100755 --- a/buildpkg.sh +++ b/buildpkg.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018, 2019 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,97 +20,9 @@ source "$TOP/lib/common.sh" logmust check_running_system -function merge_with_upstream() { - local upstream_ref="refs/heads/upstream-HEAD" - - logmust cd "$WORKDIR/repo" - check_git_ref "$upstream_ref" "refs/heads/repo-HEAD" - - logmust git checkout -q repo-HEAD - - if git merge-base --is-ancestor "$upstream_ref" HEAD; then - echo "NOTE: $PACKAGE is already up-to-date with upstream." - return 0 - fi - - echo "Running: git merge --no-edit $upstream_ref" - if git merge --no-edit --no-stat "$upstream_ref"; then - echo "git merge succeeded" - logmust touch "$WORKDIR/repo-updated" - return 0 - else - echo "git merge failed" - logmust git merge --abort - return 1 - fi -} - -# -# Inititalize Delphix git repository from a source package. -# -function inititalize_from_upstream_source_package() { - check_env UPSTREAM_SOURCE_PACKAGE - - # - # Fetch the source package into source/ - # - logmust mkdir "$WORKDIR/source" - logmust cd "$WORKDIR/source" - logmust apt-get source "$UPSTREAM_SOURCE_PACKAGE" - - # - # Create initial repository from the package source. - # Both repo-HEAD and upstream-HEAD point to the same commit. - # - logmust cd "$WORKDIR" - logmust mv source/"$UPSTREAM_SOURCE_PACKAGE"*/ repo - logmust cd "$WORKDIR/repo" - logmust git init - logmust git checkout -b repo-HEAD - logmust git add -f . - logmust generate_commit_message_from_dsc - logmust git commit -F "$WORKDIR/commit-message" - logmust git branch upstream-HEAD - - logmust touch "$WORKDIR/upstream-updated" - logmust touch "$WORKDIR/repo-updated" -} - -# -# Inititalize Delphix git repository from an upstream git repository. -# -function inititalize_from_upstream_git() { - check_env UPSTREAM_GIT_URL UPSTREAM_GIT_BRANCH - - logmust mkdir "$WORKDIR/repo" - logmust cd "$WORKDIR/repo" - logmust git init - logmust git remote add upstream "$UPSTREAM_GIT_URL" - logmust git fetch upstream "$UPSTREAM_GIT_BRANCH" - - logmust git branch repo-HEAD FETCH_HEAD - logmust git branch upstream-HEAD FETCH_HEAD - - logmust git checkout -q repo-HEAD - - logmust touch "$WORKDIR/upstream-updated" - logmust touch "$WORKDIR/repo-updated" -} - -function inititalize_from_upstream() { - if [[ -n "$UPSTREAM_GIT_URL" ]]; then - logmust inititalize_from_upstream_git - elif [[ -n "$UPSTREAM_SOURCE_PACKAGE" ]]; then - logmust inititalize_from_upstream_source_package - else - die "$PACKAGE/config.sh must contain either" \ - "UPSTREAM_SOURCE_PACKAGE or UPSTREAM_GIT_URL/BRANCH." - fi -} - function usage() { [[ $# != 0 ]] && echo "$(basename "$0"): $*" - echo "Usage: $(basename "$0") [-i | -u [-M]] [-ch] [-g pkg_git_url]" + echo "Usage: $(basename "$0") [-ch] [-g pkg_git_url]" echo " [-b pkg_git_branch] [-v pkg_version] [-r pkg_revision]" echo " package" echo "" @@ -121,16 +33,9 @@ function usage() { echo " and then build it." echo " Options:" echo "" - echo " -i Create initial repo from an upstream git repo or" - echo " source package. Conflicts with -u." - echo " -u Update upstream branch and merge main branch with" - echo " upstream. Build only if main branch has changed." - echo " Conflicts with -i." - echo " -M When passed with -u, only update upstream branch and" - echo " never attempt to build." - echo " -c Call the checkstyle hook after fetching package." echo " -g override default git url for the package." echo " -b override default git branch for the package." + echo " -c also run package's checkstyle hook." echo " -v override default version for package." echo " -r override default revision for package." echo " -h display this message and exit." @@ -143,22 +48,14 @@ unset PARAM_PACKAGE_GIT_BRANCH unset PARAM_PACKAGE_VERSION unset PARAM_PACKAGE_REVISION -export DO_UPDATE_PACKAGE=false - do_checkstyle=false -do_initialize=false -do_merge=true -while getopts ':b:cg:hik:Mr:uv:' c; do +while getopts ':b:cg:hr:v:' c; do case "$c" in g) export PARAM_PACKAGE_GIT_URL="$OPTARG" ;; b) export PARAM_PACKAGE_GIT_BRANCH="$OPTARG" ;; v) export PARAM_PACKAGE_VERSION="$OPTARG" ;; r) export PARAM_PACKAGE_REVISION="$OPTARG" ;; - k) export TARGET_PLATFORMS="$OPTARG" ;; c) do_checkstyle=true ;; - i) do_initialize=true ;; - u) DO_UPDATE_PACKAGE=true ;; - M) do_merge=false ;; h) usage >&2 ;; *) usage "illegal option -- $OPTARG" >&2 ;; esac @@ -168,9 +65,6 @@ shift $((OPTIND - 1)) [[ $# -gt 1 ]] && usage "too many arguments" >&2 PACKAGE=$1 -$DO_UPDATE_PACKAGE && $do_initialize && usage "-i and -u are exclusive" >&2 -! $do_merge && ! $DO_UPDATE_PACKAGE && usage "-M requires -u" >&2 - logmust check_package_exists "$PACKAGE" # @@ -187,55 +81,25 @@ echo_bold "====================================================================" echo "" logmust load_package_config "$PACKAGE" - -logmust sudo rm -rf "$WORKDIR" -logmust mkdir "$WORKDIR" +logmust create_workdir logmust mkdir "$WORKDIR/artifacts" -if $do_initialize; then - logmust inititalize_from_upstream - echo_success "Repository initialized from upstream in $WORKDIR/repo" - exit 0 -fi - logmust cd "$WORKDIR" stage fetch -if $DO_UPDATE_PACKAGE; then - logmust cd "$WORKDIR" - logmust touch "$WORKDIR/updating-upstream" - type -t update_upstream >/dev/null || - die "$PACKAGE: Hook 'update_upstream()' not found!" - stage update_upstream - logmust rm "$WORKDIR/updating-upstream" - - if ! $do_merge; then - echo_bold "Not attempting to merge with upstream since" \ - "-M is set." - exit 0 - fi - - logmust touch "$WORKDIR/merging" - logmust merge_with_upstream - logmust rm "$WORKDIR/merging" - if [[ ! -f "$WORKDIR/repo-updated" ]]; then - echo_bold "Not building package $PACKAGE since we are doing" \ - "an update but the repo is already up-to-date" - exit 0 - fi -fi +logmust cd "$WORKDIR" +stage fetch_dependencies logmust cd "$WORKDIR" stage prepare -logmust touch "$WORKDIR/building" if $do_checkstyle; then logmust cd "$WORKDIR" stage checkstyle fi + logmust cd "$WORKDIR" stage build -logmust rm "$WORKDIR/building" logmust cd "$WORKDIR" stage store_build_info @@ -246,8 +110,3 @@ stage post_build_checks echo_success "Package $PACKAGE has been built successfully." echo "Build products are in $WORKDIR/artifacts" echo "" - -if $DO_UPDATE_PACKAGE; then - echo_success "Auto-merge with upstream performed" \ - "successfully in $WORKDIR/repo" -fi diff --git a/checkupdates.sh b/checkupdates.sh new file mode 100755 index 0000000..8ff0590 --- /dev/null +++ b/checkupdates.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +source "$TOP/lib/common.sh" + +logmust check_running_system + +function usage() { + [[ $# != 0 ]] && echo "$(basename "$0"): $*" + echo "Usage: $(basename "$0") " + echo "" + echo " This script checks if upstream updates are available for the" + echo " target package. It returns succesfully whether or not there are" + echo " updates available. If either the upstream can be updated or the" + echo " active branch can be merged with a previously updated upstream" + echo " then the file workdir/update-available will be created." + echo "" + echo " -h display this message and exit." + echo "" + exit 2 +} + +while getopts ':h' c; do + case "$c" in + h) usage >&2 ;; + *) usage "illegal option -- $OPTARG" >&2 ;; + esac +done +shift $((OPTIND - 1)) +[[ $# -lt 1 ]] && usage "package argument missing" >&2 +[[ $# -gt 1 ]] && usage "too many arguments" >&2 +PACKAGE=$1 + +logmust check_package_exists "$PACKAGE" + +# +# If the script is called manually, we set it here. +# +DEFAULT_REVISION="${DEFAULT_REVISION:-$(default_revision)}" +logmust determine_default_git_branch + +logmust load_package_config "$PACKAGE" +logmust create_workdir + +# +# Set DO_UPDATE_PACKAGE to true so that the fetch stage fetches both the +# target branch as well as the upstream branch. +# +export DO_UPDATE_PACKAGE=true +logmust cd "$WORKDIR" +stage fetch + +stage update_upstream +logmust is_merge_needed +merge_needed="$_RET" +$merge_needed && echo "Merge with upstream is needed." + +echo "" + +if [[ -f "$WORKDIR/upstream-updated" ]] || $merge_needed; then + logmust touch "$WORKDIR/update-available" + echo_success "Package $PACKAGE has updates available." +else + echo_bold "Package $PACKAGE is already up-to-date." +fi diff --git a/default-package-config.sh b/default-package-config.sh index 442d325..3a95072 100644 --- a/default-package-config.sh +++ b/default-package-config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,12 +27,8 @@ function fetch() { logmust fetch_repo_from_git } -function store_build_info() { - if [[ -d "$WORKDIR/repo/.git" ]]; then - logmust store_git_info - else - echo "No build info available" >"$WORKDIR/build_info" - fi +function merge_with_upstream() { + logmust merge_with_upstream_default } # @@ -93,9 +89,17 @@ function kernel_build() { # the end of the new one to maintain the mapping between # Canonical's releases and our releases. # - local canonical_abinum delphix_abinum - canonical_abinum=$(fakeroot debian/rules printenv | grep abinum | cut -d= -f2 | tr -d '[:space:]') + local canonical_abinum delphix_abinum kernel_release kernel_version + canonical_abinum=$(fakeroot debian/rules printenv | grep -E '^abinum ' | cut -d= -f2 | tr -d '[:space:]') delphix_abinum="dlpx-$(date -u +"%Y%m%dt%H%M%S")-$(git rev-parse --short HEAD)-${canonical_abinum}" + kernel_release=$(fakeroot debian/rules printenv | grep -E '^release ' | cut -d= -f2 | tr -d '[:space:]') + + # + # We record the kernel version into a file. This field is consumed + # by other kernel packages, such as zfs, during their build. + # + kernel_version="${kernel_release}-${delphix_abinum}-${platform}" + echo "$kernel_version" >"$WORKDIR/artifacts/KERNEL_VERSION" # # skipdbg=false @@ -106,8 +110,11 @@ function kernel_build() { # any intention and logic to provide signatures for now # we set it to false to avoid any misconfigurations down # the line. + # disable_d_i=true + # This prevents udeb packages from being built as they are + # not consumed by the Delphix Appliance. # - local debian_rules_args="skipdbg=false uefi_signed=false abinum=${delphix_abinum} ${debian_rules_extra_args}" + local debian_rules_args="skipdbg=false uefi_signed=false disable_d_i=true flavours=$platform abinum=${delphix_abinum} ${debian_rules_extra_args}" # # Clean up everything generated so far and recreate the @@ -132,10 +139,16 @@ function kernel_build() { local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}" - logmust fakeroot debian/rules "binary-${platform}" ${debian_rules_args} + logmust fakeroot debian/rules "binary" ${debian_rules_args} logmust cd "$WORKDIR" logmust mv ./*deb "artifacts/" + + # + # Make sure that we recorded the kernel version properly by checking + # one of the .debs produced + # + logmust test -f "artifacts/linux-image-${kernel_version}_"*.deb } # @@ -195,7 +208,7 @@ function kernel_update_upstream() { # out the latest upstream tag to sync with. # local kernel_version abinum - logmust get_kernel_for_platform "${platform}" + logmust get_kernel_version_for_platform_from_apt "${platform}" kernel_version=$(echo "$_RET" | cut -d '-' -f 1) abinum=$(echo "$_RET" | cut -d '-' -f 2) @@ -258,8 +271,60 @@ function kernel_update_upstream() { local upstream_tag upstream_tag=$(echo "${upstream_tag_info}" | awk -F / '{print $3}') [[ -z "${upstream_tag}" ]] && die "could not extract upstream tag name from the tag info" - echo "note: upstream tag: ${upstream_tag}" - logmust git fetch upstream "${upstream_tag}" + + logmust git fetch upstream "+refs/tags/${upstream_tag}:refs/tags/${upstream_tag}" + + local upstream_tag_commit + upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}")" || + die "couldn't get commit of tag ${upstream_tag}" + echo "note: upstream tag: ${upstream_tag}, commit ${upstream_tag_commit}" + + # + # Check if the commit of the latest tag from upstream matches + # what we have cached in our repository at upstreams/, + # which we fetch to upstream-HEAD. + # + local local_upstream_commit + local_upstream_commit=$(git rev-parse upstream-HEAD) + [[ -z "${local_upstream_commit}" ]] && die "could not find upstream-HEAD's commit" + echo "note: upstreams/${DEFAULT_GIT_BRANCH} commit: ${local_upstream_commit}" + + if [[ "${upstream_tag_commit}" == "${local_upstream_commit}" ]]; then + echo "NOTE: upstream for $PACKAGE is already up-to-date." + else + logmust git reset --hard "refs/tags/${upstream_tag}" + echo "NOTE: upstream updated to refs/tags/${upstream_tag}" + + # + # Store name of upstream tag so that we can push it to our + # repository for reference purposes. + # + echo "refs/tags/${upstream_tag}" >"$WORKDIR/upstream-tag" || + die "failed to write to $WORKDIR/upstream-tag" + + logmust touch "$WORKDIR/upstream-updated" + fi + + logmust cd "$WORKDIR" +} + +# +# This merges local changes in repo-HEAD with upstream changes in upstream-HEAD. +# As opposed to the default merge function merge_with_upstream_default(), this +# uses git cherry-pick to rebase our changes on top of the upstream changes. +# +function kernel_merge_with_upstream() { + local repo_ref="refs/heads/repo-HEAD" + local upstream_ref="refs/heads/upstream-HEAD" + + logmust cd "$WORKDIR/repo" + + check_git_ref "$upstream_ref" "$repo_ref" + + if git merge-base --is-ancestor "$upstream_ref" "$repo_ref"; then + echo "NOTE: $PACKAGE is already up-to-date with upstream." + return 0 + fi # # Ensure that there is a commit marking the start of @@ -276,29 +341,20 @@ function kernel_update_upstream() { [[ -z "${dlpx_patch_end}" ]] && die "could not find repo-HEAD's head commit" # - # Compare that commit with the head commit of the - # upstream tag. If the commits are the same then - # there is nothing for us to do as we are using - # the most up-to-date tag as the base for our set - # of patches. On the other hand, if the commits - # differ then it means that the upstream has been - # updated, at which point we need to cherry-pick - # our patches on top of the new upstream. + # We rebase all the Delphix commits on top of the new upstream-HEAD + # by using git cherry-pick. Note that we also save the previous + # tip of the active branch to repo-HEAD-saved as this reference will be + # checked later by push-merge.sh. # - local upstream_head_commit - upstream_head_commit=$(git rev-parse upstream-HEAD) - [[ -z "${upstream_head_commit}" ]] && die "could not find upstream-HEAD's head commit" - if [[ "${current_ubuntu_commit}" == "${upstream_head_commit}" ]]; then - echo "NOTE: upstream for $PACKAGE is already up-to-date." - else - # shellcheck disable=SC2086 - logmust git cherry-pick ${dlpx_patch_start}^..${dlpx_patch_end} + logmust git branch repo-HEAD-saved repo-HEAD + logmust git branch -D repo-HEAD + logmust git checkout -q -b repo-HEAD upstream-HEAD - logmust touch "$WORKDIR/upstream-updated" - fi + # shellcheck disable=SC2086 + logmust git cherry-pick ${dlpx_patch_start}^..${dlpx_patch_end} - logmust cd "$WORKDIR" + logmust touch "$WORKDIR/repo-updated" } function post_build_checks() { diff --git a/jenkins-buildlist.sh b/jenkins-buildlist.sh deleted file mode 100755 index dfcd72d..0000000 --- a/jenkins-buildlist.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -# -# Copyright 2018, 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -source "$TOP/lib/common.sh" - -logmust check_running_system - -function usage() { - [[ $# != 0 ]] && echo "$(basename "$0"): $*" - echo "Usage: $(basename "$0")" - echo "" - echo " This is a wrapper script that is meant to be called from" - echo " Jenkins. It consumes and processes environment variables" - echo " passed from Jenkins and call 'buildlist.sh '." - echo "" - exit 2 -} - -# -# BUILDER_CUSTOM_ENV is meant to be used to pass GIT_URL, GIT_BRANCH, VERSION -# and REVISION values for specific packages, by setting _ -# variables. See get_package_config_from_env() in lib/common.sh for more info. -# -# e.g.: -# CLOUD_INIT_GIT_BRANCH=feature-branch-1 -# CONNSTAT_GIT_URL=github.com/connstat-developer/connstat.git -# -function parse_custom_env() { - local allowed_vars=() - local prefix - local pkg - - # - # Build a list of allowed custom environment variables. - # - for pkg in "${PACKAGES[@]}"; do - get_package_prefix "$pkg" - prefix="$_RET" - for suffix in GIT_URL GIT_BRANCH VERSION REVISION; do - allowed_vars+=("${prefix}_${suffix}") - done - done - - # - # Parse each line in the custom env and check if it matches any of - # the allowed variables. - # - local found - while IFS= read -r line; do - # trim whitespace - line=$(echo "$line" | sed 's/^\s*//;s/\s*$//') - [[ -z "$line" ]] && continue - - if [[ "$line" =~ ([^=]+)=.* ]]; then - var="${BASH_REMATCH[1]}" - found=false - for allowed_var in "${allowed_vars[@]}"; do - if [[ "$allowed_var" == "$var" ]]; then - found=true - break - fi - done - $found || die "Parsing BUILDER_CUSTOM_ENV: '$var'" \ - "is not an allowed environment variable." - logmust export "$line" - else - die "Parsing BUILDER_CUSTOM_ENV: invalid entry '$line'" - fi - done < <(printf '%s\n' "$BUILDER_CUSTOM_ENV") -} - -[[ $# -eq 0 ]] || usage "takes no arguments." >&2 - -# -# Validate the list of packages to build. -# -check_env BUILD_LIST -logmust get_package_list_file "build" "$BUILD_LIST" -pkg_list_file="$_RET" -logmust read_package_list "$pkg_list_file" -PACKAGES=("${_RET_LIST[@]}") - -if [[ -n "$BUILDER_CUSTOM_ENV" ]]; then - logmust parse_custom_env -fi - -if [[ -n "$SINGLE_PACKAGE_NAME" ]]; then - logmust check_package_exists "$SINGLE_PACKAGE_NAME" - # - # Make sure that the package is actually part of the BUILD_LIST. - # - found=false - for pkg in "${PACKAGES[@]}"; do - if [[ "$pkg" == "$SINGLE_PACKAGE_NAME" ]]; then - found=true - break - fi - done - $found || die "Package SINGLE_PACKAGE_NAME=$SINGLE_PACKAGE_NAME is not" \ - "in package list '$BUILD_LIST'" - - # - # The following env parameters are propagated from jenkins: - # - # SINGLE_PACKAGE_GIT_URL, SINGLE_PACKAGE_GIT_BRANCH, - # SINGLE_PACKAGE_VERSION, SINGLE_PACKAGE_REVISION - # - # We make sure those variables are applied to package - # SINGLE_PACKAGE_NAME by copying values of SINGLE_PACKAGE_ into - # _. See comment for parse_custom_env() above. - # - logmust get_package_prefix "$SINGLE_PACKAGE_NAME" - prefix="$_RET" - echo_bold "Setting ${prefix}_ variables since" \ - "SINGLE_PACKAGE_NAME=$SINGLE_PACKAGE_NAME ..." - if [[ -n "$SINGLE_PACKAGE_GIT_URL" ]]; then - var="${prefix}_GIT_URL" - logmust export "${var}=$SINGLE_PACKAGE_GIT_URL" - fi - if [[ -n "$SINGLE_PACKAGE_GIT_BRANCH" ]]; then - var="${prefix}_GIT_BRANCH" - logmust export "${var}=$SINGLE_PACKAGE_GIT_BRANCH" - fi - if [[ -n "$SINGLE_PACKAGE_VERSION" ]]; then - var="${prefix}_VERSION" - logmust export "${var}=$SINGLE_PACKAGE_VERSION" - fi - if [[ -n "$SINGLE_PACKAGE_REVISION" ]]; then - var="${prefix}_REVISION" - logmust export "${var}=$SINGLE_PACKAGE_REVISION" - fi -fi - -logmust cd "$TOP" -logmust ./setup.sh -logmust ./buildlist.sh "$BUILD_LIST" diff --git a/jenkins-updatelist.sh b/jenkins-updatelist.sh deleted file mode 100755 index 49fa4b2..0000000 --- a/jenkins-updatelist.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -source "$TOP/lib/common.sh" - -logmust check_running_system - -function usage() { - [[ $# != 0 ]] && echo "$(basename "$0"): $*" - echo "Usage: $(basename "$0")" - echo "" - echo " This is a wrapper script that is meant to be called from" - echo " Jenkins. It consumes and processes environment variables" - echo " passed from Jenkins and call 'updatelist.sh '." - echo "" - exit 2 -} - -[[ $# -eq 0 ]] || usage "takes no arguments." >&2 - -# -# Validate the list of packages to update and make sure the GIT_DRY_RUN -# environment variable is passed. -# -check_env UPDATE_LIST GIT_DRY_RUN -logmust get_package_list_file "update" "$UPDATE_LIST" - -if [[ "$GIT_DRY_RUN" == "false" ]]; then - dry_run='' -else - dry_run='-n' -fi - -logmust cd "$TOP" -logmust ./setup.sh -logmust ./updatelist.sh $dry_run "$UPDATE_LIST" diff --git a/lib/common.sh b/lib/common.sh index 8cc6658..f181ea2 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -19,12 +19,25 @@ export _RET export _RET_LIST export DEBIAN_FRONTEND=noninteractive -# TODO: allow updating upstream for other branches than master -export REPO_UPSTREAM_BRANCH="upstreams/master" export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle" +# +# Used when fetching artifacts for external dependencies. Can be overridden +# for testing purposes to use jenkins-ops. instead. +# +export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}" +export _BASE_S3_URL="s3://snapshot-de-images/builds/${JENKINS_OPS_DIR}/devops-gate/master" + export UBUNTU_DISTRIBUTION="bionic" +# +# We currently support getting the linux kernel from 3 different sources: +# 1. "delphix": building it from code +# 2. "archive": dowloading from apt +# 3. "prebuilt": pre-built kernel stored in artifactory +# +export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="archive" + # shellcheck disable=SC2086 function enable_colors() { [[ -t 1 ]] && flags="" || flags="-T xterm" @@ -165,25 +178,6 @@ function check_git_ref() { done } -function query_git_credentials() { - if [[ -n "$PUSH_GIT_USER" ]] && [[ -n "$PUSH_GIT_PASSWORD" ]]; then - return 0 - fi - - if [[ ! -t 1 ]]; then - die "PUSH_GIT_USER and PUSH_GIT_PASSWORD environment" \ - "variables must be set to a user that has" \ - "push permissions for the target repository." - fi - - echo "Please enter git credentials for pushing to repository." - read -r -p "User: " PUSH_GIT_USER - read -r -s -p "Password: " PUSH_GIT_PASSWORD - echo "" - export PUSH_GIT_USER - export PUSH_GIT_PASSWORD -} - # # execute a hook from a package's config.sh # @@ -209,7 +203,16 @@ function reset_package_config_variables() { local hook local var - for hook in prepare fetch build checkstyle update_upstream; do + local hooks=" + prepare + fetch + build + checkstyle + update_upstream + merge_with_upstream + " + + for hook in $hooks; do unset "$hook" done @@ -229,6 +232,7 @@ function reset_package_config_variables() { WORKDIR PKGDIR PACKAGE_PREFIX + FORCE_PUSH_ON_UPDATE SKIP_COPYRIGHTS_CHECK " @@ -446,6 +450,17 @@ function get_package_config_from_env() { echo_bold "------------------------------------------------------------" } +function create_workdir() { + check_env WORKDIR + logmust sudo rm -rf "$WORKDIR" + logmust mkdir "$WORKDIR" + logmust rm -f "$TOP/workdir" + logmust ln -s "$WORKDIR" "$TOP/workdir" +} + +# +# apt install packages. +# function install_pkgs() { for attempt in {1..3}; do echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \ @@ -458,6 +473,9 @@ function install_pkgs() { die "apt-get install failed after $attempt attempts" } +# +# Install build dependencies listed in the debian/control file of the package. +# function install_build_deps_from_control_file() { logmust pushd "$WORKDIR/repo" logmust sudo env DEBIAN_FRONTEND=noninteractive mk-build-deps --install \ @@ -466,6 +484,9 @@ function install_build_deps_from_control_file() { logmust popd } +# +# Returns a list of all known packages in _RET_LIST. +# function list_all_packages() { local pkg @@ -479,6 +500,9 @@ function list_all_packages() { done } +# +# Read a package-list file and return listed packages in _RET_LIST. +# function read_package_list() { local file="$1" @@ -500,28 +524,6 @@ function read_package_list() { done <"$file" || die "Failed to read package list: $file" } -function get_package_list_file() { - local list_type="$1" - local list_name="$2" - - if [[ "$list_type" != build ]] && [[ "$list_type" != update ]]; then - die "Invalid list type '$list_type'" - fi - - _RET="$TOP/package-lists/${list_type}/${list_name}.pkgs" - if [[ ! -f "$_RET" ]]; then - echo_error "Invalid $list_type package list '$list_name'" - echo_error "See lists in $TOP/package-lists/${list_type}/." - echo_error "Choose one of:" - cd "$TOP/package-lists/${list_type}/" || - die "failed to cd to $TOP/package-lists/${list_type}/" - for list in *.pkgs; do - echo_error " ${list%.pkgs}" - done - die - fi -} - # # List all target kernel packages. By default, it returns all the kernel # flavors supported and built by linux-pkg, however this can be overridden @@ -544,7 +546,6 @@ function list_linux_kernel_packages() { _RET_LIST+=("linux-kernel-$kernel") done fi - return 0 } @@ -557,19 +558,36 @@ function install_shfmt() { echo "shfmt version $(shfmt -version) is installed." } +# +# Install kernel headers packages for all target kernels. +# The kernel packages are fetched from S3. +# function install_kernel_headers() { logmust determine_target_kernels - check_env KERNEL_VERSIONS + check_env KERNEL_VERSIONS DEPDIR - local kernel - local headers_pkgs="" + logmust list_linux_kernel_packages + # Note: linux packages returned in _RET_LIST - for kernel in $KERNEL_VERSIONS; do - headers_pkgs="$headers_pkgs linux-headers-$kernel" + # + # On some platforms there are 2 headers packages and both must be + # installed. Here's an example on AWS: + # - linux-headers-5.3.0-1030-aws_5.3.0-1030.32~18.04.1_amd64.deb + # - linux-aws-5.3-headers-5.3.0-1030_5.3.0-1030.32~18.04.1_all.deb + # + local pkg + for pkg in "${_RET_LIST[@]}"; do + logmust install_pkgs "$DEPDIR/$pkg/"*-headers-*.deb done - # shellcheck disable=SC2086 - logmust install_pkgs $headers_pkgs + # + # Verify that headers are installed for all kernel versions + # stored in KERNEL_VERSIONS + # + local kernel + for kernel in $KERNEL_VERSIONS; do + logmust dpkg-query -l "linux-headers-$kernel" >/dev/null + done } function default_revision() { @@ -590,11 +608,58 @@ function default_revision() { echo "delphix-$(date '+%Y.%m.%d.%H')" } +# +# Fetch artifacts from S3 for all packages listed in PACKAGE_DEPENDENCIES which +# is defined in the package's config. +# +function fetch_dependencies() { + export DEPDIR="$WORKDIR/dependencies" + logmust mkdir "$DEPDIR" + logmust cd "$DEPDIR" + + if [[ -z "$PACKAGE_DEPENDENCIES" ]]; then + echo "Package has no linux-pkg dependencies to fetch." + return + fi + + local base_url="$_BASE_S3_URL/linux-pkg/$DEFAULT_GIT_BRANCH/build-package" + + local bucket="${_BASE_S3_URL#s3://}" + bucket=${bucket%%/*} + + local dep s3urlvar s3url + for dep in $PACKAGE_DEPENDENCIES; do + echo "Fetching artifacts for dependency '$dep' ..." + get_package_prefix "$dep" + s3urlvar="${_RET}_S3_URL" + if [[ -n "${!s3urlvar}" ]]; then + s3url="${!s3urlvar}" + echo "S3 URL of package dependency '$dep' provided" \ + "externally" + echo "$s3urlvar=$s3url" + else + s3url="$base_url/$dep/post-push" + (logmust aws s3 cp --only-show-errors "$s3url/latest" .) || + die "Artifacts for dependency '$dep' missing." \ + "Dependency must be built first." + logmust cat latest + s3url="s3://$bucket/$(cat latest)" + logmust rm latest + fi + [[ "$s3url" != */ ]] && s3url="$s3url/" + logmust mkdir "$dep" + logmust aws s3 ls "$s3url" + logmust aws s3 cp --only-show-errors --recursive "$s3url" "$dep/" + echo_bold "Fetched artifacts for '$dep' from $s3url" + PACKAGE_DEPENDENCIES_METADATA="${PACKAGE_DEPENDENCIES_METADATA}$dep: $s3url\\n" + done +} + # # Fetch package repository into $WORKDIR/repo # function fetch_repo_from_git() { - check_env PACKAGE_GIT_URL PACKAGE_GIT_BRANCH + check_env PACKAGE_GIT_URL PACKAGE_GIT_BRANCH DEFAULT_GIT_BRANCH logmust mkdir "$WORKDIR/repo" logmust cd "$WORKDIR/repo" @@ -605,15 +670,17 @@ function fetch_repo_from_git() { # main branch and the upstream branch with their histories. # Otherwise just get the latest commit of the main branch. # - if $DO_UPDATE_PACKAGE; then - check_env REPO_UPSTREAM_BRANCH + if [[ "$DO_UPDATE_PACKAGE" == "true" ]]; then logmust git fetch --no-tags "$PACKAGE_GIT_URL" \ "+$PACKAGE_GIT_BRANCH:repo-HEAD" logmust git fetch --no-tags "$PACKAGE_GIT_URL" \ - "+$REPO_UPSTREAM_BRANCH:upstream-HEAD" + "+upstreams/$DEFAULT_GIT_BRANCH:upstream-HEAD" + logmust git show-ref repo-HEAD + logmust git show-ref upstream-HEAD else logmust git fetch --no-tags "$PACKAGE_GIT_URL" \ "+$PACKAGE_GIT_BRANCH:repo-HEAD" --depth=1 + logmust git show-ref repo-HEAD fi logmust git checkout repo-HEAD @@ -631,6 +698,11 @@ function generate_commit_message_from_dsc() { cat "$dsc" >>"$WORKDIR/commit-message" } +# +# Fetches branch upstreams/ from our default repository into local +# upstream-HEAD branch, then attempts to update it with changes from +# the source package UPSTREAM_SOURCE_PACKAGE, fetched from apt. +# function update_upstream_from_source_package() { check_env PACKAGE_GIT_BRANCH UPSTREAM_SOURCE_PACKAGE @@ -669,6 +741,7 @@ function update_upstream_from_source_package() { else logmust generate_commit_message_from_dsc logmust git commit -F "$WORKDIR/commit-message" + logmust git show-ref upstream-HEAD logmust touch "$WORKDIR/upstream-updated" fi @@ -676,6 +749,11 @@ function update_upstream_from_source_package() { logmust cd "$WORKDIR" } +# +# Fetches branch upstreams/ from our default repository into local +# upstream-HEAD branch, then attempts to update it with changes from +# the remote repository specified by UPSTREAM_GIT_URL and UPSTREAM_GIT_BRANCH. +# function update_upstream_from_git() { check_env UPSTREAM_GIT_URL UPSTREAM_GIT_BRANCH logmust cd "$WORKDIR/repo" @@ -705,6 +783,7 @@ function update_upstream_from_git() { # then we definitely want to be notified. # logmust git merge --no-edit --ff-only --no-stat FETCH_HEAD + logmust git show-ref upstream-HEAD logmust touch "$WORKDIR/upstream-updated" fi @@ -712,6 +791,115 @@ function update_upstream_from_git() { logmust cd "$WORKDIR" } +# +# Returns true if upstreams/ needs to be merged into for the +# active package, where is the branch being updated, i.e. +# DEFAULT_GIT_BRANCH. +# +function is_merge_needed() { + local repo_ref="refs/heads/repo-HEAD" + local upstream_ref="refs/heads/upstream-HEAD" + + logmust pushd "$WORKDIR/repo" + check_git_ref "$upstream_ref" "$repo_ref" + + if git merge-base --is-ancestor "$upstream_ref" "$repo_ref"; then + echo "Upstream is already merged into repo-HEAD" + _RET=false + else + _RET=true + fi + logmust popd +} + +# +# Default function for merging upstreams/ into , where +# is the branch being updated, i.e. DEFAULT_GIT_BRANCH. Note that this function +# does not actually look at the upstream repository itself, but relies on +# local branches repo-HEAD and upstream-HEAD to be present. +# +# If merge was needed, file $WORKDIR/repo-updated is created and previous tip +# of is saved in repo-HEAD-saved. The repo-updated file lets the +# caller (typically Jenkins) know if a merge was necessary. The repo-HEAD-saved +# ref should be compared to the remote branch when it is time to push the +# merge; if they differ it means that the remote branch was modified and +# so the merge should be aborted -- this can happen if a PR was merged by a +# developer while auto-update was running. +# +function merge_with_upstream_default() { + local repo_ref="refs/heads/repo-HEAD" + local upstream_ref="refs/heads/upstream-HEAD" + + logmust cd "$WORKDIR/repo" + check_git_ref "$upstream_ref" "$repo_ref" + + logmust git checkout -q repo-HEAD + + if git merge-base --is-ancestor "$upstream_ref" HEAD; then + echo "NOTE: $PACKAGE is already up-to-date with upstream." + return 0 + fi + + # + # Do a backup of the repo-HEAD branch so that it can be compared to the + # remote when time comes to do a push. + # + logmust git branch repo-HEAD-saved + + logmust git merge --no-edit --no-stat "$upstream_ref" + logmust git show-ref repo-HEAD + logmust touch "$WORKDIR/repo-updated" +} + +# +# Check if git credentials are set for pushing update. If running in +# interactive mode, it will prompt the user for credentials if they are not +# provided in env. +# +function check_git_credentials_set() { + if [[ -z "$PUSH_GIT_USER" ]] || [[ -z "$PUSH_GIT_PASSWORD" ]]; then + if [[ -t 1 ]]; then + if [[ "$DRYRUN" == "false" ]]; then + echo_bold "WARNING: this is NOT a dry-run, you are pushing to" \ + "a production branch" + fi + echo "Please enter git credentials to push to remote ($DEFAULT_PACKAGE_GIT_URL)." + read -r -p "Username: " PUSH_GIT_USER + read -r -s -p "Password: " PUSH_GIT_PASSWORD + export PUSH_GIT_USER + export PUSH_GIT_PASSWORD + else + die "PUSH_GIT_USER and PUSH_GIT_PASSWORD must be set." + fi + fi +} + +# +# Push a local ref to a remote ref of the default remote repository for the +# package. +# +function push_to_remote() { + local local_ref="$1" + local remote_ref="$2" + local force="${3:-false}" + + local flags="" + $force && flags="-f" + + logmust check_git_credentials_set + + check_env DEFAULT_PACKAGE_GIT_URL PUSH_GIT_USER PUSH_GIT_PASSWORD + local git_url_with_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:${PUSH_GIT_PASSWORD}@}" + local git_url_with_fake_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:@}" + + logmust cd "$WORKDIR/repo" + check_git_ref "$local_ref" + + echo "RUNNING: git push $flags $git_url_with_fake_creds $local_ref:$remote_ref" + git push $flags "$git_url_with_creds" "$local_ref:$remote_ref" || + die "push failed" +} + # # Creates a new changelog entry for the package with the appropriate fields. # If no changelog file exists, source package name can be passed in first arg. @@ -733,6 +921,10 @@ function set_changelog() { fi } +# +# Default dpkg_buildpackage function for building packages. Before running the +# build, it updates the version of the package in the changelog. +# function dpkg_buildpackage_default() { logmust cd "$WORKDIR/repo" logmust set_changelog @@ -741,25 +933,11 @@ function dpkg_buildpackage_default() { logmust mv ./*deb artifacts/ } -# -# Store some metadata about what was this package built from. When running -# buildlist.sh, build_info for all packages is ingested by the metapackage -# and installed into /lib/delphix-buildinfo/.info. -# -function store_git_info() { - logmust pushd "$WORKDIR/repo" - echo "Git hash: $(git rev-parse HEAD)" >"$WORKDIR/build_info" || - die "storing git info failed" - echo "Git repo: $PACKAGE_GIT_URL" >>"$WORKDIR/build_info" - echo "Git branch: $PACKAGE_GIT_BRANCH" >>"$WORKDIR/build_info" - logmust popd -} - # # Returns the default (usually latest) kernel version for a given platform. # Result is placed into _RET. # -function get_kernel_for_platform() { +function get_kernel_version_for_platform_from_apt() { local platform="$1" local package @@ -796,52 +974,114 @@ function get_kernel_for_platform() { fi } +# +# Given a kernel version, fetch all necessary linux kernel packages +# into WORKDIR/artifacts. Also store kernel version into KERNEL_VERSION. +# +function fetch_kernel_from_apt_for_version() { + local kernel_version="$1" + + logmust cd "$WORKDIR/artifacts" + logmust apt-get download \ + "linux-image-${kernel_version}" \ + "linux-image-${kernel_version}-dbgsym" \ + "linux-modules-${kernel_version}" \ + "linux-headers-${kernel_version}" \ + "linux-tools-${kernel_version}" + + # + # Fetch direct dependencies of the downloaded debs. Some of those + # dependencies have a slightly different naming scheme than the other + # kernel packages. + # + local deb dep deps + for deb in *.deb; do + deps=$(dpkg-deb -f "$deb" Depends | tr -d ' ' | tr ',' ' ') || + die "failed to get dependencies for $deb" + for dep in $deps; do + case "$dep" in + *-headers-* | *-tools-*) + logmust apt-get download "$dep" + ;; + esac + done + done + + echo "$kernel_version" >KERNEL_VERSION +} + +# +# Find latest linux kernel available in apt for the given platform, and +# download all the necessary linux-kernel packages. +# +function fetch_kernel_from_apt_for_platform() { + local platform="$1" + + local kernel_version + logmust get_kernel_version_for_platform_from_apt "$platform" + kernel_version="$_RET" + + logmust fetch_kernel_from_apt_for_version "$kernel_version" +} + +# +# Fetch linux kernel packages from apt for the given kernel version. Also +# fetch the pre-built linux-modules package from artifactory. The pre-built +# package should have the same name as the one downloaded from apt but +# a higher revision number so that it will be picked over the default one +# downloaded from apt during the build of the appliance. +# +function fetch_kernel_from_artifactory() { + local kernel_version="$1" + local artifactory_deb="$2" + + logmust fetch_kernel_from_apt_for_version "$kernel_version" + + local url="http://artifactory.delphix.com/artifactory" + url="$url/linux-pkg/linux-prebuilt/${artifactory_deb}" + + logmust cd "$WORKDIR/artifacts" + logmust wget -nv "$url" +} + # # Determine which kernel versions to build modules for and store # the value into KERNEL_VERSIONS, unless it is already set. # -# We determine the target kernel versions based on the value passed for -# TARGET_PLATFORMS. Here is a list of accepted values for TARGET_PLATFORMS: -# a) : to build for all supported platforms -# b) "aws gcp ...": to build for the default kernel version of those platforms. -# c) "4.15.0-1010-aws ...": to build for specific kernel versions -# d) mix of b) and c) +# We determine the target kernel versions based on the kernel package +# dependencies fetched through fetch_dependencies(). # function determine_target_kernels() { if [[ -n "$KERNEL_VERSIONS" ]]; then - echo "Kernel versions to use to build modules:" - echo " $KERNEL_VERSIONS" + echo_bold "Kernel versions to use to build modules:" + echo_bold " $KERNEL_VERSIONS" return 0 fi - local supported_platforms="generic aws gcp azure oracle" - local platform + [[ -n "$DEPDIR" ]] || die "determine_target_kernels() can only be" \ + "called after fetch_dependencies() stage has run." - if [[ -z "$TARGET_PLATFORMS" ]]; then - echo "TARGET_PLATFORMS not set, defaulting to: $supported_platforms" - TARGET_PLATFORMS="$supported_platforms" - fi + logmust list_linux_kernel_packages + # note: list of kernel packages returned in _RET_LIST - local kernel - for kernel in $TARGET_PLATFORMS; do - for platform in $supported_platforms; do - if [[ "$kernel" == "$platform" ]]; then - logmust get_kernel_for_platform "$platform" - kernel="$_RET" - break - fi - done + local pkg kernel + for pkg in "${_RET_LIST[@]}"; do + logmust test -d "$DEPDIR/$pkg" # - # Check that the target kernel is valid + # When Linux kernel packages are built, they must store the + # kernel version into a file named 'KERNEL_VERSION'. # - apt-cache show "linux-image-${kernel}" >/dev/null 2>&1 || - die "Invalid target kernel '$kernel'" - + (logmust test -f "$DEPDIR/$pkg/KERNEL_VERSION") || + die "KERNEL_VERSION file missing from dependency '$pkg'" + kernel="$(cat "$DEPDIR/$pkg/KERNEL_VERSION")" + [[ -n "$kernel" ]] || die "invalid value '$kernel'" \ + "in $DEPDIR/$pkg/KERNEL_VERSION" KERNEL_VERSIONS="$KERNEL_VERSIONS $kernel" + KERNEL_VERSIONS_METADATA="${KERNEL_VERSIONS_METADATA}${pkg}: ${kernel}\\n" done - echo "Kernel versions to use to build modules:" - echo " $KERNEL_VERSIONS" + echo_bold "Kernel versions to use to build modules:" + echo_bold " $KERNEL_VERSIONS" } # @@ -854,3 +1094,51 @@ function install_gcc8() { logmust sudo update-alternatives --install /usr/bin/gcc gcc \ /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8 } + +# +# Store git-related build info for the package after the build is done. +# Note that some of this metadata is used by the Jenkins build so be careful +# when modifying it. +# +function store_git_info() { + logmust pushd "$WORKDIR/repo" + local git_hash + git_hash="$(git rev-parse HEAD)" || die "Failed retrieving git hash" + echo "$git_hash" >"$WORKDIR/artifacts/GIT_HASH" + + cat <<-EOF >>"$WORKDIR/artifacts/BUILD_INFO" + Git hash: $git_hash + Git repo: $PACKAGE_GIT_URL + Git branch: $PACKAGE_GIT_BRANCH + EOF + logmust popd +} + +# +# Store build info metadata for the package after the build is done. +# Note that some of this metadata is used by the Jenkins build so be careful +# when modifying it. +# +function store_build_info() { + if [[ -d "$WORKDIR/repo/.git" ]]; then + logmust store_git_info + fi + + if [[ -n "$KERNEL_VERSIONS_METADATA" ]]; then + echo -ne "$KERNEL_VERSIONS_METADATA" >"$WORKDIR/artifacts/KERNEL_VERSIONS" || + die 'Failed to store kernel versions metadata' + fi + + if [[ -n "$PACKAGE_DEPENDENCIES_METADATA" ]]; then + echo -ne "$PACKAGE_DEPENDENCIES_METADATA" >"$WORKDIR/artifacts/PACKAGE_DEPENDENCIES" || + die 'Failed to store package dependencies metadata' + fi + + if [[ -f "$TOP/PACKAGE_MIRROR_URL_MAIN" ]]; then + logmust cp "$TOP/PACKAGE_MIRROR_URL_MAIN" "$WORKDIR/artifacts/" + fi + + if [[ -f "$TOP/PACKAGE_MIRROR_URL_SECONDARY" ]]; then + logmust cp "$TOP/PACKAGE_MIRROR_URL_SECONDARY" "$WORKDIR/artifacts/" + fi +} diff --git a/package-lists/auto-merge-blacklist.pkgs b/package-lists/auto-merge-blacklist.pkgs deleted file mode 100644 index 1268325..0000000 --- a/package-lists/auto-merge-blacklist.pkgs +++ /dev/null @@ -1,5 +0,0 @@ -# -# Packages in this list will not be auto-merged with upstream, even if they -# are listed in updatelist.pkgs. A package should be added here only if the -# latest version of an upstream package breaks the delphix appliance. -# diff --git a/package-lists/build/adoptopenjdk.pkgs b/package-lists/build/adoptopenjdk.pkgs deleted file mode 100644 index 36c66ee..0000000 --- a/package-lists/build/adoptopenjdk.pkgs +++ /dev/null @@ -1,7 +0,0 @@ -# -# This list is responsible of building adoptopenjdk. We first need to build -# our modified make-jpkg package. -# - -make-jpkg -adoptopenjdk diff --git a/package-lists/build/kernel.pkgs b/package-lists/build/kernel.pkgs deleted file mode 100644 index 50be040..0000000 --- a/package-lists/build/kernel.pkgs +++ /dev/null @@ -1,14 +0,0 @@ -# -# This build list includes all the packages that are build for specific -# kernel versions. -# - -# Note: The following packages should be built first because other packages -# depend on them being built. -# - zfs is required by grub2 and recovery-environment -zfs - -connstat -delphix-kernel -grub2 -recovery-environment diff --git a/package-lists/build/td-agent.pkgs b/package-lists/build/td-agent.pkgs deleted file mode 100644 index 92b741b..0000000 --- a/package-lists/build/td-agent.pkgs +++ /dev/null @@ -1,8 +0,0 @@ -# -# Since building td-agent is currently quite unreliable as it depends on -# lots of external dependencies that are out of our control, we do not want -# to fail our build when some of those dependencies are broken. As such, -# we build td-agent in a separate build list and include a pre-built package -# in the main list. -# -td-agent diff --git a/package-lists/build/userland.pkgs b/package-lists/build/userland.pkgs deleted file mode 100644 index 0220d2f..0000000 --- a/package-lists/build/userland.pkgs +++ /dev/null @@ -1,33 +0,0 @@ -# -# List of misc userland packages to be included in the Delphix Appliance. -# - -# Note: The following packages should be built first because other packages -# depend on them being built. -# - bcc is required by bpftrace -# - java8 is required by the saml app -bcc -java8 -make-jpkg -adoptopenjdk -libkdumpfile - -delphix-sso-app -bpftrace -challenge-response -cloud-init -crash -crash-python -crypt-blowfish -delphix-platform -drgn -gdb-python -makedumpfile -nfs-utils -python-rtslib-fb -savedump -sdb -targetcli-fb -performance-diagnostics -ptools -td-agent-prebuilt diff --git a/package-lists/update/userland.pkgs b/package-lists/update/userland.pkgs deleted file mode 100644 index c9b588c..0000000 --- a/package-lists/update/userland.pkgs +++ /dev/null @@ -1,23 +0,0 @@ -# -# All those packages will be auto-updated by the linux-pkg-update/userland job. -# Note that if you want to keep a package's upstream branch to be updated -# but prevent it from being merged, add the package to -# auto-merge-blacklist.pkg. -# - -# Note: we do not auto-update bcc for now as upstream tends to introduce -# changes that break bpftrace. Instead, we manually update to tagged versions. -# In the future we may want to automatically determine the latest tag and -# auto-update to that. -#bcc - -bpftrace -cloud-init -crash -drgn -libkdumpfile -makedumpfile -nfs-utils -python-rtslib-fb -targetcli-fb -grub2 diff --git a/package-lists/update/zfs.pkgs b/package-lists/update/zfs.pkgs deleted file mode 100644 index e5c56b7..0000000 --- a/package-lists/update/zfs.pkgs +++ /dev/null @@ -1,6 +0,0 @@ -# -# ZFS is configured to be updated every time we detect an upstream change, -# so we provide a separate update list for it. -# - -zfs diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh index 84eab59..eb14a27 100755 --- a/packages/adoptopenjdk/config.sh +++ b/packages/adoptopenjdk/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,10 +23,7 @@ tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz" jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" function prepare() { - if ! ls "$TOP/packages/make-jpkg/tmp/artifacts/"*deb >/dev/null 2>&1; then - echo_bold "custom java-package not installed. Building package 'make-jpkg' first." - logmust "$TOP/buildpkg.sh" make-jpkg - fi + logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb } function fetch() { @@ -53,13 +50,6 @@ function build() { # the Linux-pkg bundle. # logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'" - # - # Install the Java package on this system so that other linux-pkg - # packages can use it. - # - logmust install_pkgs "$WORKDIR/artifacts/"*.deb -} -function store_build_info() { - echo "Tar file: $tarfile" >"$WORKDIR/build_info" + echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO" } diff --git a/packages/bcc/config.sh b/packages/bcc/config.sh index 26b4bf0..c7717bc 100644 --- a/packages/bcc/config.sh +++ b/packages/bcc/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,9 +32,6 @@ function build() { PACKAGE_VERSION=$(dpkg-parsechangelog | sed -rne 's,^Version: (.*),\1,p') logmust dpkg_buildpackage_default - - # Install libbcc which is required to build bpftrace - logmust install_pkgs "$WORKDIR/artifacts"/libbcc_*.deb } function update_upstream() { diff --git a/packages/bpftrace/config.sh b/packages/bpftrace/config.sh index ef2f636..52c5345 100644 --- a/packages/bpftrace/config.sh +++ b/packages/bpftrace/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,11 +24,7 @@ UPSTREAM_GIT_URL="https://github.com/iovisor/bpftrace.git" UPSTREAM_GIT_BRANCH="master" function prepare() { - if ! dpkg-query --show libbcc >/dev/null 2>&1; then - echo_bold "libbcc not installed. Building package 'bcc' first." - logmust "$TOP/buildpkg.sh" bcc - fi - + logmust install_pkgs "$DEPDIR"/bcc/libbcc_*.deb logmust install_build_deps_from_control_file } diff --git a/packages/cloud-init/config.sh b/packages/cloud-init/config.sh index c40fb0b..35b8b77 100644 --- a/packages/cloud-init/config.sh +++ b/packages/cloud-init/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2019 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/connstat/config.sh b/packages/connstat/config.sh index 8bb5e3a..cc53ed2 100644 --- a/packages/connstat/config.sh +++ b/packages/connstat/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/crash-python/config.sh b/packages/crash-python/config.sh index 9449bb2..f3a58ee 100644 --- a/packages/crash-python/config.sh +++ b/packages/crash-python/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/crypt-blowfish/config.sh b/packages/crypt-blowfish/config.sh index 1f020ad..cc3e3a9 100644 --- a/packages/crypt-blowfish/config.sh +++ b/packages/crypt-blowfish/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/delphix-kernel/config.sh b/packages/delphix-kernel/config.sh index f464dfd..e7a7848 100644 --- a/packages/delphix-kernel/config.sh +++ b/packages/delphix-kernel/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/delphix-platform/config.sh b/packages/delphix-platform/config.sh index 69a6cc0..d070ffc 100644 --- a/packages/delphix-platform/config.sh +++ b/packages/delphix-platform/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 956fdb6..d3fcd2b 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,20 +17,16 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/saml-app.git" -JDK_PATH_FILE="$TOP/packages/adoptopenjdk/tmp/artifacts/JDK_PATH" PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { - java_package_exists=$(dpkg-query --show adoptopenjdk-java8-jdk >/dev/null 2>&1) - if [[ ! $java_package_exists && ! -f $JDK_PATH_FILE ]]; then - echo_bold "java8 not installed. Building package 'adoptopenjdk' first." - logmust "$TOP/buildpkg.sh" adoptopenjdk - fi + logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb } function build() { local java_home - java_home=$(cat "$JDK_PATH_FILE") + java_home=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || + die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" logmust cd "$WORKDIR/repo" logmust sudo ./gradlew "-Dorg.gradle.java.home=$java_home" distDeb logmust sudo mv ./build/distributions/*deb "$WORKDIR/artifacts/" diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index 737263a..d0d1ada 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,11 +29,7 @@ function prepare() { # drgn itself, but it is a hard requirement in our use-case as # we do want to use drgn for kdump-compressed crash dumps. # - if ! dpkg-query --show libkdumpfile >/dev/null 2>&1; then - echo_bold "libkdumpfile not installed. Building package 'libkdumpfile' first." - logmust "$TOP/buildpkg.sh" libkdumpfile - fi - + logmust install_pkgs "$DEPDIR"/libkdumpfile/*.deb logmust install_build_deps_from_control_file } diff --git a/packages/gdb-python/config.sh b/packages/gdb-python/config.sh index aea4a7c..fbe6ee9 100644 --- a/packages/gdb-python/config.sh +++ b/packages/gdb-python/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh index 254fe76..8807d7f 100644 --- a/packages/grub2/config.sh +++ b/packages/grub2/config.sh @@ -28,13 +28,9 @@ SKIP_COPYRIGHTS_CHECK=true # Install build dependencies for the package. # function prepare() { - if ! dpkg-query --show libzfslinux-dev >/dev/null 2>&1; then - echo_bold "libzfs not installed. Building package 'zfs' first." - logmust "$TOP/buildpkg.sh" zfs - fi - + # Install libzfs which is required to build grub + logmust install_pkgs "$DEPDIR"/zfs/{libnvpair1linux,libuutil1linux,libzfs2linux,libzpool2linux,libzfslinux-dev}_*.deb logmust install_build_deps_from_control_file - return } # @@ -54,5 +50,4 @@ function build() { # function update_upstream() { logmust update_upstream_from_git - return } diff --git a/packages/java8/config.sh b/packages/java8/config.sh index 39f98d3..5d03bae 100644 --- a/packages/java8/config.sh +++ b/packages/java8/config.sh @@ -45,13 +45,6 @@ function build() { # the Linux-pkg bundle. # logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'" - # - # Install the Java package on this system so that other linux-pkg - # packages can use it. - # - logmust install_pkgs "$WORKDIR/artifacts/"*.deb -} -function store_build_info() { - echo "Tar file: $tarfile" >"$WORKDIR/build_info" + echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO" } diff --git a/packages/libkdumpfile/config.sh b/packages/libkdumpfile/config.sh index 7710e2d..a42c82e 100644 --- a/packages/libkdumpfile/config.sh +++ b/packages/libkdumpfile/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,9 +28,6 @@ function prepare() { function build() { logmust dpkg_buildpackage_default - - # Install libkdumpfile, it's needed to build drgn - logmust install_pkgs "$WORKDIR/artifacts"/*.deb } function update_upstream() { diff --git a/build-info-pkg/debian/rules b/packages/linux-kernel-aws/config.archive.sh old mode 100755 new mode 100644 similarity index 70% rename from build-info-pkg/debian/rules rename to packages/linux-kernel-aws/config.archive.sh index a4c3e1d..c8b581d --- a/build-info-pkg/debian/rules +++ b/packages/linux-kernel-aws/config.archive.sh @@ -1,6 +1,6 @@ -#!/usr/bin/make -f +#!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck disable=SC2034 -%: - dh $@ +DEFAULT_PACKAGE_GIT_URL="none" -override_dh_auto_test: - # - # Don't run 'make test' during the build step; we'll enforce - # testing via another mechanism, so running it during package - # builds is unnecessary. - # +function fetch() { + # Nothing to do + return +} +function build() { + logmust fetch_kernel_from_apt_for_platform aws +} diff --git a/packages/linux-kernel-aws/config.delphix.sh b/packages/linux-kernel-aws/config.delphix.sh new file mode 100644 index 0000000..5d7a2cc --- /dev/null +++ b/packages/linux-kernel-aws/config.delphix.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-aws.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-aws/+git/bionic" +# Note: UPSTREAM_GIT_BRANCH is not used here +UPSTREAM_GIT_BRANCH="none" + +# +# Force push required when syncing with upstream because we perform a rebase. +# +FORCE_PUSH_ON_UPDATE=true + +function prepare() { + logmust kernel_prepare +} + +function build() { + logmust kernel_build "aws" +} + +function update_upstream() { + logmust kernel_update_upstream "aws" +} + +function merge_with_upstream() { + logmust kernel_merge_with_upstream +} diff --git a/build-info-pkg/debian/control.in b/packages/linux-kernel-aws/config.prebuilt.sh similarity index 64% rename from build-info-pkg/debian/control.in rename to packages/linux-kernel-aws/config.prebuilt.sh index a8f07dc..55067be 100644 --- a/build-info-pkg/debian/control.in +++ b/packages/linux-kernel-aws/config.prebuilt.sh @@ -1,5 +1,6 @@ +#!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,14 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck disable=SC2034 -Source: @@PACKAGE@@ -Section: metapackages -Priority: optional -Maintainer: Delphix Engineering -Build-Depends: debhelper (>= 10) -Standards-Version: 4.1.2 +DEFAULT_PACKAGE_GIT_URL="none" -Package: @@PACKAGE@@ -Architecture: any -Description: Provides build metadata for packages built by linux-pkg. +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_artifactory "5.3.0-1033-aws" \ + "6.0.5.0/dx1/linux-modules-5.3.0-1033-aws_5.3.0-1033.dx1_amd64.deb" +} diff --git a/packages/linux-kernel-aws/config.sh b/packages/linux-kernel-aws/config.sh index b42417b..3a8c5ea 100644 --- a/packages/linux-kernel-aws/config.sh +++ b/packages/linux-kernel-aws/config.sh @@ -15,20 +15,19 @@ # limitations under the License. # -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-aws.git" - -UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-aws/+git/bionic" -UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" - -function prepare() { - kernel_prepare -} - -function build() { - kernel_build "aws" -} +# +# We currently support getting the linux kernel from 3 different sources: +# 1. Building it from code: see config.delphix.sh +# 2. Dowloading from apt: see config.archive.sh +# 3. Pre-built kernel stored in artifactory: see config.prebuilt.sh +# -function update_upstream() { - kernel_update_upstream "aws" -} +linux_package_source="${LINUX_KERNEL_PACKAGE_SOURCE:-$DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE}" +case "$linux_package_source" in +delphix | archive | prebuilt) + logmust source "${BASH_SOURCE%/*}/config.${linux_package_source}.sh" + ;; +default) + die "invalid linux-kernel package source '$linux_package_source'" + ;; +esac diff --git a/build-info-pkg/clean.sh b/packages/linux-kernel-azure/config.archive.sh old mode 100755 new mode 100644 similarity index 67% rename from build-info-pkg/clean.sh rename to packages/linux-kernel-azure/config.archive.sh index 7108de5..e84cb7e --- a/build-info-pkg/clean.sh +++ b/packages/linux-kernel-azure/config.archive.sh @@ -1,6 +1,6 @@ -#!/bin/bash -eu +#!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck disable=SC2034 -cd "$(dirname "${BASH_SOURCE[0]}")" +DEFAULT_PACKAGE_GIT_URL="none" -rm -rf lib -rm -rf artifacts -rm -f debian/control -rm -f debian/changelog -rm -rf debian/.debhelper/ -rm -rf debian/delphix-buildinfo* -rm -f debian/files -rm -f debian/debhelper-build-stamp +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_apt_for_platform "azure" +} diff --git a/packages/linux-kernel-azure/config.delphix.sh b/packages/linux-kernel-azure/config.delphix.sh new file mode 100644 index 0000000..a6bcc23 --- /dev/null +++ b/packages/linux-kernel-azure/config.delphix.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-azure.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-azure/+git/bionic" +# Note: UPSTREAM_GIT_BRANCH is not used here +UPSTREAM_GIT_BRANCH="none" + +# +# Force push required when syncing with upstream because we perform a rebase. +# +FORCE_PUSH_ON_UPDATE=true + +function prepare() { + logmust kernel_prepare +} + +function build() { + logmust kernel_build "azure" +} + +function update_upstream() { + logmust kernel_update_upstream "azure" +} + +function merge_with_upstream() { + logmust kernel_merge_with_upstream +} diff --git a/packages/linux-kernel-azure/config.prebuilt.sh b/packages/linux-kernel-azure/config.prebuilt.sh new file mode 100644 index 0000000..5c02477 --- /dev/null +++ b/packages/linux-kernel-azure/config.prebuilt.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="none" + +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_artifactory "5.3.0-1035-azure" \ + "6.0.5.0/dx1/linux-modules-5.3.0-1035-azure_5.3.0-1035.dx1_amd64.deb" +} diff --git a/packages/linux-kernel-azure/config.sh b/packages/linux-kernel-azure/config.sh index ddb8c34..3a8c5ea 100644 --- a/packages/linux-kernel-azure/config.sh +++ b/packages/linux-kernel-azure/config.sh @@ -15,20 +15,19 @@ # limitations under the License. # -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-azure.git" - -UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-azure/+git/bionic" -UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" - -function prepare() { - kernel_prepare -} - -function build() { - kernel_build "azure" -} +# +# We currently support getting the linux kernel from 3 different sources: +# 1. Building it from code: see config.delphix.sh +# 2. Dowloading from apt: see config.archive.sh +# 3. Pre-built kernel stored in artifactory: see config.prebuilt.sh +# -function update_upstream() { - kernel_update_upstream "azure" -} +linux_package_source="${LINUX_KERNEL_PACKAGE_SOURCE:-$DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE}" +case "$linux_package_source" in +delphix | archive | prebuilt) + logmust source "${BASH_SOURCE%/*}/config.${linux_package_source}.sh" + ;; +default) + die "invalid linux-kernel package source '$linux_package_source'" + ;; +esac diff --git a/packages/linux-kernel-gcp/config.archive.sh b/packages/linux-kernel-gcp/config.archive.sh new file mode 100644 index 0000000..f39a400 --- /dev/null +++ b/packages/linux-kernel-gcp/config.archive.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="none" + +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_apt_for_platform "gcp" +} diff --git a/packages/linux-kernel-gcp/config.delphix.sh b/packages/linux-kernel-gcp/config.delphix.sh new file mode 100644 index 0000000..342547c --- /dev/null +++ b/packages/linux-kernel-gcp/config.delphix.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-gcp.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-gcp/+git/bionic" +# Note: UPSTREAM_GIT_BRANCH is not used here +UPSTREAM_GIT_BRANCH="none" + +# +# Force push required when syncing with upstream because we perform a rebase. +# +FORCE_PUSH_ON_UPDATE=true + +function prepare() { + logmust kernel_prepare +} + +function build() { + logmust kernel_build "gcp" +} + +function update_upstream() { + logmust kernel_update_upstream "gcp" +} + +function merge_with_upstream() { + logmust kernel_merge_with_upstream +} diff --git a/packages/linux-kernel-gcp/config.prebuilt.sh b/packages/linux-kernel-gcp/config.prebuilt.sh new file mode 100644 index 0000000..0ac0f4b --- /dev/null +++ b/packages/linux-kernel-gcp/config.prebuilt.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="none" + +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_artifactory "5.4.0-1021-gcp" \ + "6.0.5.0/dx1/linux-modules-5.4.0-1021-gcp_5.4.0-1021.dx1_amd64.deb" +} diff --git a/packages/linux-kernel-gcp/config.sh b/packages/linux-kernel-gcp/config.sh index 2ba303b..3a8c5ea 100644 --- a/packages/linux-kernel-gcp/config.sh +++ b/packages/linux-kernel-gcp/config.sh @@ -15,20 +15,19 @@ # limitations under the License. # -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-gcp.git" - -UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-gcp/+git/bionic" -UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" - -function prepare() { - kernel_prepare -} - -function build() { - kernel_build "gcp" -} +# +# We currently support getting the linux kernel from 3 different sources: +# 1. Building it from code: see config.delphix.sh +# 2. Dowloading from apt: see config.archive.sh +# 3. Pre-built kernel stored in artifactory: see config.prebuilt.sh +# -function update_upstream() { - kernel_update_upstream "gcp" -} +linux_package_source="${LINUX_KERNEL_PACKAGE_SOURCE:-$DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE}" +case "$linux_package_source" in +delphix | archive | prebuilt) + logmust source "${BASH_SOURCE%/*}/config.${linux_package_source}.sh" + ;; +default) + die "invalid linux-kernel package source '$linux_package_source'" + ;; +esac diff --git a/packages/linux-kernel-generic/config.archive.sh b/packages/linux-kernel-generic/config.archive.sh new file mode 100644 index 0000000..124f19f --- /dev/null +++ b/packages/linux-kernel-generic/config.archive.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="none" + +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_apt_for_platform "generic" +} diff --git a/packages/linux-kernel-generic/config.delphix.sh b/packages/linux-kernel-generic/config.delphix.sh new file mode 100644 index 0000000..4db55af --- /dev/null +++ b/packages/linux-kernel-generic/config.delphix.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-generic.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/bionic" +# Note: UPSTREAM_GIT_BRANCH is not used here +UPSTREAM_GIT_BRANCH="none" + +# +# Force push required when syncing with upstream because we perform a rebase. +# +FORCE_PUSH_ON_UPDATE=true + +function prepare() { + logmust kernel_prepare +} + +function build() { + logmust kernel_build "generic" +} + +function update_upstream() { + logmust kernel_update_upstream "generic" +} + +function merge_with_upstream() { + logmust kernel_merge_with_upstream +} diff --git a/packages/linux-kernel-generic/config.prebuilt.sh b/packages/linux-kernel-generic/config.prebuilt.sh new file mode 100644 index 0000000..ed700f7 --- /dev/null +++ b/packages/linux-kernel-generic/config.prebuilt.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="none" + +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_artifactory "5.4.0-42-generic" \ + "6.0.5.0/dx1/linux-modules-5.4.0-42-generic_5.4.0-42.dx1_amd64.deb" +} diff --git a/packages/linux-kernel-generic/config.sh b/packages/linux-kernel-generic/config.sh index baa0708..3a8c5ea 100644 --- a/packages/linux-kernel-generic/config.sh +++ b/packages/linux-kernel-generic/config.sh @@ -15,26 +15,19 @@ # limitations under the License. # -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-generic.git" - -UPSTREAM_GIT_URL="https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/bionic" -UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" - -function prepare() { - kernel_prepare -} - -function build() { - # - # flavours=generic - # By default the generic kernel variant from Canonical - # builds both the generic and the low-latency kernel. - # We don't care about the latter. - # - kernel_build "generic" "flavours=generic" -} +# +# We currently support getting the linux kernel from 3 different sources: +# 1. Building it from code: see config.delphix.sh +# 2. Dowloading from apt: see config.archive.sh +# 3. Pre-built kernel stored in artifactory: see config.prebuilt.sh +# -function update_upstream() { - kernel_update_upstream "generic" -} +linux_package_source="${LINUX_KERNEL_PACKAGE_SOURCE:-$DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE}" +case "$linux_package_source" in +delphix | archive | prebuilt) + logmust source "${BASH_SOURCE%/*}/config.${linux_package_source}.sh" + ;; +default) + die "invalid linux-kernel package source '$linux_package_source'" + ;; +esac diff --git a/packages/linux-kernel-oracle/config.archive.sh b/packages/linux-kernel-oracle/config.archive.sh new file mode 100644 index 0000000..6c283a3 --- /dev/null +++ b/packages/linux-kernel-oracle/config.archive.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="none" + +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_apt_for_platform "oracle" +} diff --git a/packages/linux-kernel-oracle/config.delphix.sh b/packages/linux-kernel-oracle/config.delphix.sh new file mode 100644 index 0000000..0bd2763 --- /dev/null +++ b/packages/linux-kernel-oracle/config.delphix.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# shellcheck disable=SC2034 +DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-oracle.git" + +UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oracle/+git/bionic" +# Note: UPSTREAM_GIT_BRANCH is not used here +UPSTREAM_GIT_BRANCH="none" + +# +# Force push required when syncing with upstream because we perform a rebase. +# +FORCE_PUSH_ON_UPDATE=true + +function prepare() { + logmust kernel_prepare +} + +function build() { + logmust kernel_build "oracle" +} + +function update_upstream() { + logmust kernel_update_upstream "oracle" +} + +function merge_with_upstream() { + logmust kernel_merge_with_upstream +} diff --git a/packages/linux-kernel-oracle/config.prebuilt.sh b/packages/linux-kernel-oracle/config.prebuilt.sh new file mode 100644 index 0000000..82115eb --- /dev/null +++ b/packages/linux-kernel-oracle/config.prebuilt.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="none" + +function fetch() { + # Nothing to do + return +} + +function build() { + logmust fetch_kernel_from_artifactory "5.4.0-1021-oracle" \ + "6.0.5.0/dx1/linux-modules-5.4.0-1021-oracle_5.4.0-1021.dx1_amd64.deb" +} diff --git a/packages/linux-kernel-oracle/config.sh b/packages/linux-kernel-oracle/config.sh index 85b1a10..3a8c5ea 100644 --- a/packages/linux-kernel-oracle/config.sh +++ b/packages/linux-kernel-oracle/config.sh @@ -15,20 +15,19 @@ # limitations under the License. # -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/linux-kernel-oracle.git" - -UPSTREAM_GIT_URL="https://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oracle/+git/bionic" -UPSTREAM_GIT_BRANCH="@PLACEHOLDER-WORKAROUND@" - -function prepare() { - kernel_prepare -} - -function build() { - kernel_build "oracle" -} +# +# We currently support getting the linux kernel from 3 different sources: +# 1. Building it from code: see config.delphix.sh +# 2. Dowloading from apt: see config.archive.sh +# 3. Pre-built kernel stored in artifactory: see config.prebuilt.sh +# -function update_upstream() { - kernel_update_upstream "oracle" -} +linux_package_source="${LINUX_KERNEL_PACKAGE_SOURCE:-$DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE}" +case "$linux_package_source" in +delphix | archive | prebuilt) + logmust source "${BASH_SOURCE%/*}/config.${linux_package_source}.sh" + ;; +default) + die "invalid linux-kernel package source '$linux_package_source'" + ;; +esac diff --git a/packages/make-jpkg/config.sh b/packages/make-jpkg/config.sh index 6aab3f8..42c1cb5 100644 --- a/packages/make-jpkg/config.sh +++ b/packages/make-jpkg/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,11 +32,6 @@ function build() { fi logmust dpkg_buildpackage_default - - # java-package supporting adoptopenjdk needs to be installed to - # create jdk debian package from jdk tarball when building adoptopenjdk - # package - logmust install_pkgs "$WORKDIR/artifacts"/*.deb } function update_upstream() { diff --git a/packages/makedumpfile/config.sh b/packages/makedumpfile/config.sh index 03676b3..398105b 100644 --- a/packages/makedumpfile/config.sh +++ b/packages/makedumpfile/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/ptools/config.sh b/packages/ptools/config.sh index 2c63700..ca7edbc 100644 --- a/packages/ptools/config.sh +++ b/packages/ptools/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2019 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/recovery-environment/config.sh b/packages/recovery-environment/config.sh index 1ca3394..74e1c04 100644 --- a/packages/recovery-environment/config.sh +++ b/packages/recovery-environment/config.sh @@ -20,16 +20,10 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/recovery-environment.git" DEFAULT_PACKAGE_VERSION=1.0.0 PACKAGE_DEPENDENCIES="zfs" -ZFS_DEB_PATH="$TOP/packages/zfs/tmp/artifacts" function prepare() { - if [ ! "$(ls -A "$ZFS_DEB_PATH")" ]; then - logmust "$TOP/buildpkg.sh" zfs - fi - logmust install_build_deps_from_control_file logmust mkdir "$WORKDIR/repo/external-debs/" - logmust cp "$ZFS_DEB_PATH/"*.deb "$WORKDIR/repo/external-debs/" - return + logmust cp "$DEPDIR/zfs/"*.deb "$WORKDIR/repo/external-debs/" } function build() { diff --git a/packages/targetcli-fb/config.sh b/packages/targetcli-fb/config.sh index 7b6f406..4d06dd3 100644 --- a/packages/targetcli-fb/config.sh +++ b/packages/targetcli-fb/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ PACKAGE_DEPENDENCIES="python-rtslib-fb" UPSTREAM_SOURCE_PACKAGE=targetcli-fb function prepare() { + logmust install_pkgs "$DEPDIR"/python-rtslib-fb/python3-rtslib-fb*.deb logmust install_build_deps_from_control_file } diff --git a/packages/td-agent-prebuilt/config.sh b/packages/td-agent-prebuilt/config.sh index 4fef2cc..8e5e215 100644 --- a/packages/td-agent-prebuilt/config.sh +++ b/packages/td-agent-prebuilt/config.sh @@ -18,11 +18,10 @@ DEFAULT_PACKAGE_GIT_URL=none SKIP_COPYRIGHTS_CHECK=true -package="td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb" function fetch() { logmust cd "$WORKDIR/artifacts" - + local package="td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb" local url="http://artifactory.delphix.com/artifactory" logmust wget -nv "$url/linux-pkg/td-agent/$package" -O "$package" diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index 6d3f695..3140045 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2019 Delphix +# Copyright 2019, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -163,9 +163,6 @@ function build() { done logmust cd "$WORKDIR" logmust mv "all-packages/"*.deb "artifacts/" - - # Install libzfs which is required to build grub - logmust install_pkgs "$WORKDIR/artifacts"/{libnvpair1linux,libuutil1linux,libzfs2linux,libzpool2linux,libzfslinux-dev}_*.deb } function update_upstream() { diff --git a/push-merge.sh b/push-merge.sh new file mode 100755 index 0000000..ec60e89 --- /dev/null +++ b/push-merge.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +source "$TOP/lib/common.sh" + +logmust check_running_system + +function usage() { + [[ $# != 0 ]] && echo "$(basename "$0"): $*" + echo "Usage: $(basename "$0") " + echo "" + echo " Push code that was previously merged. sync-with-upstream.sh must" + echo " already have been run. Before pushing the merge, it will first" + echo " check that the target branch has not been modified since the merge" + echo " was performed, and fail if it did." + echo "" + echo " As a safety check, DRYRUN environment variable must be set to" + echo " 'false'." + echo "" + echo " -h display this message and exit." + echo "" + exit 2 +} + +while getopts ':h' c; do + case "$c" in + h) usage >&2 ;; + *) usage "illegal option -- $OPTARG" >&2 ;; + esac +done +shift $((OPTIND - 1)) +[[ $# -lt 1 ]] && usage "package argument missing" >&2 +[[ $# -gt 1 ]] && usage "too many arguments" >&2 +PACKAGE=$1 + +if [[ "$DRYRUN" != 'false' ]]; then + die "DRYRUN environment variable must be set to 'false'." +fi + +logmust check_package_exists "$PACKAGE" + +DEFAULT_REVISION="${DEFAULT_REVISION:-$(default_revision)}" +logmust determine_default_git_branch +logmust load_package_config "$PACKAGE" + +if [[ ! -d "$WORKDIR/repo" ]]; then + die "$WORKDIR/repo doesn't exist, have you run sync-with-upstream for" \ + "package $PACKAGE?" +fi +logmust cd "$WORKDIR/repo" + +# +# Check that the target branch has not been modified in the meanwhile. +# This is especially important for repositories that have +# FORCE_PUSH_ON_UPDATE set to true, such as the linux kernel. The file +# WORKDIR/merge-commit-outdated will be created if that is the case to let +# the caller know this is the reason the push failed. +# +set -o pipefail +echo "Running: git rev-parse refs/heads/repo-HEAD-saved" +saved_ref=$(git rev-parse refs/heads/repo-HEAD-saved) || + die "Failed to read local ref refs/heads/repo-HEAD-saved" +echo "Running: git ls-remote $DEFAULT_PACKAGE_GIT_URL refs/heads/$DEFAULT_GIT_BRANCH" +remote_ref=$(git ls-remote "$DEFAULT_PACKAGE_GIT_URL" "refs/heads/$DEFAULT_GIT_BRANCH" | + awk '{print $1}') || + die "Failed to read remote ref refs/heads/$DEFAULT_GIT_BRANCH" +set +o pipefail + +if [[ "$saved_ref" != "$remote_ref" ]]; then + touch "$WORKDIR/merge-commit-outdated" + die "Remote branch $DEFAULT_GIT_BRANCH was modified while merge" \ + "testing was being performed. Previous hash: $saved_ref," \ + "new hash: $remote_ref. Not pushing merge." +fi + +force_push="${FORCE_PUSH_ON_UPDATE:-false}" +logmust push_to_remote "refs/heads/repo-HEAD" \ + "refs/heads/$DEFAULT_GIT_BRANCH" "$force_push" + +echo_success "Merge pushed successfully for package $PACKAGE to remote" \ + "branch $DEFAULT_GIT_BRANCH" diff --git a/push-updates.sh b/push-updates.sh deleted file mode 100755 index 0beaa31..0000000 --- a/push-updates.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -source "$TOP/lib/common.sh" - -logmust check_running_system - -function usage() { - [[ $# != 0 ]] && echo "$(basename "$0"): $*" - echo "Usage: $(basename "$0") -m|-u [-ny] [-g target_git_url]" - echo " [-b target_git_branch] package" - echo "" - echo " This script pushes updates generated by 'buildpkg.sh -u' to the" - echo " default remote for the package. The updates should be present" - echo " in 'packages//tmp/repo', in branch repo-HEAD for the merge" - echo " branch and upstream-HEAD for the upstream branch." - echo " Options:" - echo "" - echo " -u push updates for the upstream branch" - echo " -m push updates for the merge branch" - echo " -g repository to push to. Defaults to value of" - echo " DEFAULT_PACKAGE_GIT_URL for the package." - echo " -b branch to push to. Defaults to 'master' when passing -m" - echo " or to $REPO_UPSTREAM_BRANCH when passing -u." - echo " -n dry-run. Pass the dry-run flag to git push (git push -n)." - echo " -y do not prompt for confirmation before pushing." - echo " -h display this message and exit." - echo "" - exit 2 -} - -unset TARGET_GIT_URL -unset TARGET_GIT_BRANCH - -do_push_merge=false -do_push_upstream=false -do_ask=true -dry_run=false -while getopts ':b:g:hmnuy' c; do - case "$c" in - b) TARGET_GIT_BRANCH="$OPTARG" ;; - g) TARGET_GIT_URL="$OPTARG" ;; - m) do_push_merge=true ;; - u) do_push_upstream=true ;; - n) dry_run=true ;; - y) do_ask=false ;; - h) usage >&2 ;; - *) usage "illegal option -- $OPTARG" >&2 ;; - esac -done -shift $((OPTIND - 1)) -[[ $# -lt 1 ]] && usage "package argument missing" >&2 -[[ $# -gt 1 ]] && usage "too many arguments" >&2 -PACKAGE=$1 - -if $do_push_merge && $do_push_upstream; then - usage "-m and -u are exclusive." >&2 -elif ! ($do_push_merge || $do_push_upstream); then - usage "must pass either -m or -u." >&2 -fi - -query_git_credentials - -logmust check_package_exists "$PACKAGE" -PKGDIR="$TOP/packages/$PACKAGE" -WORKDIR="$PKGDIR/tmp" - -if [[ -z "$TARGET_GIT_URL" ]]; then - # - # load the package's config just so that we can retrieve - # DEFAULT_PACKAGE_GIT_URL. - # - logmust load_package_config "$PACKAGE" - check_env DEFAULT_PACKAGE_GIT_URL - TARGET_GIT_URL="$DEFAULT_PACKAGE_GIT_URL" - echo "TARGET_GIT_URL set to DEFAULT_PACKAGE_GIT_URL" -fi - -if [[ -z "$TARGET_GIT_BRANCH" ]]; then - if $do_push_merge; then - TARGET_GIT_BRANCH="master" - echo "TARGET_GIT_BRANCH defaults to 'master' for merge pushes." - else - TARGET_GIT_BRANCH="$REPO_UPSTREAM_BRANCH" - echo "TARGET_GIT_BRANCH defaults to '$REPO_UPSTREAM_BRANCH' for" \ - "upstream pushes." - fi -fi - -[[ "$TARGET_GIT_URL" == https://* ]] || - die "Target git url must begin with https://" - -git_url_with_creds="${TARGET_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:${PUSH_GIT_PASSWORD}@}" -git_url_with_fake_creds="${TARGET_GIT_URL/https:\/\//https:\/\/:@}" - -if $do_push_merge; then - ref="refs/heads/repo-HEAD" - check_updated="$WORKDIR/repo-updated" -else - ref="refs/heads/upstream-HEAD" - check_updated="$WORKDIR/upstream-updated" -fi - -[[ -f "$check_updated" ]] || - die "$check_updated is missing, aborting push." - -logmust cd "$WORKDIR/repo" -check_git_ref "$ref" - -if $do_ask; then - echo_bold "WARNING: You are about to push to branch" \ - "$TARGET_GIT_BRANCH of $TARGET_GIT_URL." - read -rp "Do you want to proceed (y/[n])? " - if [[ "$REPLY" != "y" ]]; then - echo "Push aborted by user." - exit 1 - fi -fi - -flags="" -$dry_run && flags="-n" -echo "Running: git push $flags $git_url_with_fake_creds $ref:$TARGET_GIT_BRANCH" -git push $flags "$git_url_with_creds" "$ref:$TARGET_GIT_BRANCH" || - die "Push failed." diff --git a/resources/delphix-secondary-mirror.key b/resources/delphix-secondary-mirror.key new file mode 100644 index 0000000000000000000000000000000000000000..96d26c61c1e016635b3816e5ae4e284070366f94 GIT binary patch literal 2255 zcmV;=2r&1V0u2OO$L8Mw5CFrLBe~q^o`w9{E%HSdf%__gR~5lO016u!QV5$Kg={o> zxKPcMYr<3pJMcg>$sX`efXd=13k#}T$VNIG=|Cy$?)&!gkYJ?_#3Pv#Ou;QI0gE!f z6uHR|e9TS1x6F=_^`V02gEP~%%Sl>oEtbaj2keKP*E6O}*OKv_hq$`g7UbAvSsKzZ9HkhdR-+tD8oJCZS~>gP;m@ z8~b1cNF0v@1)cza;cBs>Uk_OK)m2))n1!;zqC&#osjO9{WqlNG9;4_cF?mX@2!o zdQHE9yRU-ONb3L*0RRECBSK+wX>1@va&LDaJYr#UX>2ZIa&LD)WMyn{XlZyZV{dIf zi2_ap69EVSI2IuUnTFJG+#RXVPojPJv2@%0Ve))a1p->f=HCJv0|g5S2nPZN6$%Lm z3jzcd0s{d89svRufB*^!5V3UI{bBNaQ`Hd<{x?M!wg%G-9x%`ycOCrm)E@pP+}uI4 z<@^Ff;%=FKok6-NBFOUq=dT7|T^y++#Y(yVxH5Ao1_U9sEzIvw5(6ODb|oceZvwS0vvQY zI4iOqhbtbr7*5P+CI5nSG9IJ;GK8h{Csi9nkXF8N=y;U%Z(l+{dj_Q6y8E!^-g(YxJaGDYheD3%sL71vFP6c?rB83Q(X^XdjFB)^4(K0 z;DwjCtk$@e?zeHDmJEDe7CE%|;nH%9Mh-Y4&#>l(9gbX-idRhU8F>4mpsH06r5 z0ZWN8_p$?caNW*lxP9z9NpCzdUYJl3Tm{AAqb-AB<{jGI-1V~OLDJq7hGi&C@$F%p zIB~*E+Bz;Xo$?E~mG%y=7ir+Dleq#71X{=D-vJN+#nBb6;v%DPOJ`Bfem|)d)lry4 zd;D>G;<1`N7PF)jMU82Pk*N7Gn%=b+kHJye!ueFB5sO?BQH({o4^x-%>sdA_d9Ijc-5b3VsfFUv($?B)oz>m=kw|GDYJL~R30H%lA4&Epu_lb4*i>|E<|#-p95 zjV)Qz9*{S$wa2C#{{W-ho$~jT^W%xM!Y&UUJ`96@Jzm+p*JjVA-dRbqeAY>}*tu)D zCMi!N2!nw#RrAMxM^SL(3S)&VRmcU(=P4Ato*=*&9vHu6pkgaU z=7^1h1+t;p{P@tx=z%qAA3mj7&Ib?yrRjdd?gf^(dscqotumFD0dtF3l zQZ-yncp84%$blbM%wz8sPW1E!qr9E>E+lcPoG*v&tU!#<$Nk*bV3i@NOwGe7fJ~ql zsW0U_el`LO`0oyk_n91VHEoad23~EO01*KI0f_=O1Q-Db03a421eu1^aNHfK&`+X$ z__1``{bBNaQw0KA$L8Mx8w>yn2@tV#+x=nkd{c$a5B@OOK!M*BP5lWv=52whckhVec#b&Gp;6t8_hjPmex>PO&`WI_6LdkW_}b3W&!1lH|jyY@Ai;_3$f zwlcmd1R`TrR%;Hfit$T-t0qhxlK1eSl^Ke}@OYClx@r$%g|n518eFZD9VAIeyDJ%I za-y?Vu5o@j;|q%zTFO!&78C)k`e^JIbu|E=H|I&c?DstO*!mLlLoq)g-lE`BzQ?C% zsNzcb^BJhn;XRI#4f``ACbgMmn})~^ z;ddjYIk(4>=ML)t@hbV22E`r60rF03mf`eBVS0z|M~(i2}Zc zl5yXjXHr#KZcLMEcr4-Df(+-`TPh! dPuwM`LKr8?N literal 0 HcmV?d00001 diff --git a/setup.sh b/setup.sh index f07223c..2d0b232 100755 --- a/setup.sh +++ b/setup.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,18 +25,28 @@ logmust determine_default_git_branch # mirror url is passed in, then the latest mirror snapshot is used. # configure_apt_sources() { - local package_mirror_url='' - if [[ -n "$DELPHIX_PACKAGE_MIRROR_MAIN" ]]; then - package_mirror_url="$DELPHIX_PACKAGE_MIRROR_MAIN" - else + local package_mirror_url + local primary_url="$DELPHIX_PACKAGE_MIRROR_MAIN" + local secondary_url="$DELPHIX_PACKAGE_MIRROR_SECONDARY" + + if [[ -z "$primary_url" ]] || [[ -z "$secondary_url" ]]; then local latest_url="http://linux-package-mirror.delphix.com/" latest_url+="${DEFAULT_GIT_BRANCH}/latest/" package_mirror_url=$(curl -LfSs -o /dev/null -w '%{url_effective}' \ "$latest_url" || die "Could not curl $latest_url") - - package_mirror_url+="ubuntu" + # Remove trailing slash, if present. + package_mirror_url="${package_mirror_url%/}" + [[ -z "$primary_url" ]] && primary_url="${package_mirror_url}/ubuntu" + [[ -z "$secondary_url" ]] && secondary_url="${package_mirror_url}/ppas" fi + # + # Store the package mirror in a file so that it can be added to a + # package build's metadata via store_build_info(). + # + echo "$primary_url" >"$TOP/PACKAGE_MIRROR_URL_MAIN" + echo "$secondary_url" >"$TOP/PACKAGE_MIRROR_URL_SECONDARY" + # # Remove other sources in sources.list.d if they are present. # @@ -46,18 +56,23 @@ configure_apt_sources() { ) sudo bash -c "cat <<-EOF >/etc/apt/sources.list -deb ${package_mirror_url} ${UBUNTU_DISTRIBUTION} main restricted universe multiverse -deb-src ${package_mirror_url} ${UBUNTU_DISTRIBUTION} main restricted universe multiverse + deb ${primary_url} ${UBUNTU_DISTRIBUTION} main restricted universe multiverse + deb-src ${primary_url} ${UBUNTU_DISTRIBUTION} main restricted universe multiverse + + deb ${primary_url} ${UBUNTU_DISTRIBUTION}-updates main restricted universe multiverse + deb-src ${primary_url} ${UBUNTU_DISTRIBUTION}-updates main restricted universe multiverse + + deb ${primary_url} ${UBUNTU_DISTRIBUTION}-security main restricted universe multiverse + deb-src ${primary_url} ${UBUNTU_DISTRIBUTION}-security main restricted universe multiverse -deb ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-updates main restricted universe multiverse -deb-src ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-updates main restricted universe multiverse + deb ${primary_url} ${UBUNTU_DISTRIBUTION}-backports main restricted universe multiverse + deb-src ${primary_url} ${UBUNTU_DISTRIBUTION}-backports main restricted universe multiverse -deb ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-security main restricted universe multiverse -deb-src ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-security main restricted universe multiverse + deb ${secondary_url} ${UBUNTU_DISTRIBUTION} main multiverse universe + deb ${secondary_url} ${UBUNTU_DISTRIBUTION}-updates main multiverse universe + EOF" || die "/etc/apt/sources.list could not be updated" -deb ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-backports main restricted universe multiverse -deb-src ${package_mirror_url} ${UBUNTU_DISTRIBUTION}-backports main restricted universe multiverse -EOF" || die "/etc/apt/sources.list could not be updated" + logmust sudo apt-key add "$TOP/resources/delphix-secondary-mirror.key" } logmust check_running_system diff --git a/sync-with-upstream.sh b/sync-with-upstream.sh new file mode 100755 index 0000000..1f0e206 --- /dev/null +++ b/sync-with-upstream.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +source "$TOP/lib/common.sh" + +logmust check_running_system + +function usage() { + [[ $# != 0 ]] && echo "$(basename "$0"): $*" + echo "Usage: $(basename "$0") " + echo "" + echo " Update the upstreams branch for the package and attempt to merge" + echo " active branch with the upstream. If merge succeeds, push result" + echo " to branch projects/auto-update//merging." + echo "" + echo " This script requires the DRYRUN environment variable to be set to" + echo " either 'true' or 'false', else it will refuse to push anything" + echo " upstream. If DRYRUN is set to true, the upstreams/ branch" + echo " will not be pushed and the resulting merge will be pushed to" + echo " projects/auto-update//merging-dryrun instead." + echo "" + echo " -h display this message and exit." + echo "" + exit 2 +} + +while getopts ':h' c; do + case "$c" in + h) usage >&2 ;; + *) usage "illegal option -- $OPTARG" >&2 ;; + esac +done +shift $((OPTIND - 1)) +[[ $# -lt 1 ]] && usage "package argument missing" >&2 +[[ $# -gt 1 ]] && usage "too many arguments" >&2 +PACKAGE=$1 + +logmust check_package_exists "$PACKAGE" + +logmust determine_default_git_branch + +merging_ref="refs/heads/projects/auto-update/$DEFAULT_GIT_BRANCH/merging" +if [[ "$DRYRUN" == 'true' ]]; then + merging_ref="${merging_ref}-dryrun" +elif [[ "$DRYRUN" != 'false' ]]; then + die "DRYRUN environment variable must be set to 'true' or 'false'." +fi + +logmust load_package_config "$PACKAGE" +logmust create_workdir + +# +# Set DO_UPDATE_PACKAGE to true so that the fetch stage fetches both the +# target branch as well as the upstream branch. +# +export DO_UPDATE_PACKAGE=true +logmust cd "$WORKDIR" +stage fetch + +stage update_upstream +force_push="${FORCE_PUSH_ON_UPDATE:-false}" + +if [[ -f "$WORKDIR/upstream-updated" ]]; then + if $DRYRUN; then + echo_success "Upstream updated for package $PACKAGE" \ + "but not pushed because this is a dry-run." + else + logmust push_to_remote "refs/heads/upstream-HEAD" \ + "refs/heads/upstreams/$DEFAULT_GIT_BRANCH" "$force_push" + + if [[ -f "$WORKDIR/upstream-tag" ]]; then + echo "Note: also pushing tag from upstream." + upstream_tag="$(cat "$WORKDIR/upstream-tag")" + [[ -z "$upstream_tag" ]] && + die "tag missing in $WORKDIR/upstream-tag" + logmust push_to_remote "$upstream_tag" \ + "$upstream_tag" false + fi + + echo_success "Upstream updated for package $PACKAGE." + fi +fi + +logmust cd "$WORKDIR/repo" + +stage merge_with_upstream +if [[ -f "$WORKDIR/repo-updated" ]]; then + logmust push_to_remote "refs/heads/repo-HEAD" "$merging_ref" true + echo_success "Pushed merge commit of package $PACKAGE to ref" \ + "$merging_ref of $DEFAULT_PACKAGE_GIT_URL for testing." +else + echo_bold "Package is already up-to-date." +fi diff --git a/template/config.sh b/template/config.sh index bb9a4b6..4c15fcb 100644 --- a/template/config.sh +++ b/template/config.sh @@ -61,7 +61,7 @@ function prepare() { # #logmust install_pkgs build-dep-pkg1 build-dep-pkg2 ... #logmust install_build_deps_from_control_file - return + echo 'insert code here' } # @@ -70,7 +70,7 @@ function prepare() { # function build() { # - # Those are the default functions to build the package: + # This is the default functions to build the package: # logmust dpkg_buildpackage_default } @@ -85,5 +85,5 @@ function update_upstream() { # #logmust update_upstream_from_source_package #logmust update_upstream_from_git - return + echo 'insert code here' } diff --git a/updatelist.sh b/updatelist.sh deleted file mode 100755 index 293dcc1..0000000 --- a/updatelist.sh +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash -# -# Copyright 2018, 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script first updates a list of third-party packages with upstream -# by running "buildpkg.sh -u" on each package. If updates can be merged -# cleanly and the resulting package builds, the merge is pushed to the -# package's repository (denoted by DEFAULT_PACKAGE_GIT_URL in its config.sh). -# - -TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -source "$TOP/lib/common.sh" - -logmust check_running_system - -function exit_hook() { - echo_error "Script has exited unexpectedly." - FAILURE=true - - if [[ -z "$STATUS_DIR" ]] || [[ ! -d "$STATUS_DIR" ]]; then - echo_error "Script failed during the setup phase." - return - fi - - report_status_all -} - -function record_failure() { - FAILURE=true - if $stop_on_failure; then - echo_error "Stopping on failure." - trap - EXIT - - report_status_all - exit 1 - fi -} - -function usage() { - [[ $# != 0 ]] && echo "$(basename "$0"): $*" - echo "Usage: $(basename "$0") [-hns] " - echo "" - echo "This script attempts to update all the packages in" - echo "package-lists/update/.pkgs." - echo "" - echo " -n dry-run. Pass the dry-run flag to git push (git push -n)." - echo " -s stop on failure. By default script continues when update" - echo " for a package fails." - echo " -h display this message and exit." - echo "" - exit 2 -} - -dry_run=false -stop_on_failure=false -while getopts ':hns' c; do - case "$c" in - s) stop_on_failure=true ;; - n) dry_run=true ;; - h) usage >&2 ;; - *) usage "illegal option -- $OPTARG" >&2 ;; - esac -done -shift $((OPTIND - 1)) -[[ $# -ne 1 ]] && usage "takes exactly one argument." >&2 - -pkg_list="$1" -logmust get_package_list_file "update" "$pkg_list" -pkg_list_file="$_RET" - -trap exit_hook EXIT -FAILURE=false - -$dry_run && echo "This is a dry-run, updates will NOT be pushed to remotes." - -logmust query_git_credentials - -logmust cd "$TOP" - -logmust make clean -STATUS_DIR="$TOP/update-status" -logmust mkdir "$STATUS_DIR" - -function report_status() { - local pkg - local status - - if $FAILURE; then - status="${FMT_RED}FAILURE${FMT_NF}" - else - status="${FMT_GREEN}SUCCESS${FMT_NF}" - fi - - echo "" - echo -e "${FMT_BOLD}Status Report: $status" - - # - # Return if there is nothing to report - # - [[ -f "$STATUS_DIR/upstream-pushed" ]] || - [[ -f "$STATUS_DIR/merge-pushed" ]] || - [[ -f "$STATUS_DIR/update-failed" ]] || - [[ -f "$STATUS_DIR/unexpected-failure" ]] || - return - - echo_bold "___________________________________________" - - if $dry_run; then - echo_bold "NOTE: This is a dry-run, updates are NOT pushed." - echo "" - fi - - if [[ -f "$STATUS_DIR/upstream-pushed" ]]; then - echo -e "${FMT_GREEN}Upstream updated for following" \ - "packages:${FMT_NF}" - while read -r pkg; do - echo -e "${FMT_GREEN} $pkg${FMT_NF}" - done <"$STATUS_DIR/upstream-pushed" - echo "" - fi - - if [[ -f "$STATUS_DIR/merge-pushed" ]]; then - echo -e "${FMT_GREEN}Merged following packages with" \ - "upstream:${FMT_NF}" - while read -r pkg; do - echo -e "${FMT_GREEN} $pkg${FMT_NF}" - done <"$STATUS_DIR/merge-pushed" - echo "" - fi - - if [[ -f "$STATUS_DIR/update-failed" ]]; then - echo -e "${FMT_RED}Failed to update following packages:" \ - "${FMT_NF}" - while read -r pkg; do - echo -e "${FMT_RED} $pkg${FMT_NF}" - done <"$STATUS_DIR/update-failed" - echo "" - fi - - if [[ -f "$STATUS_DIR/unexpected-failure" ]]; then - echo -e "${FMT_RED}Unexpected failure when updating following" \ - "packages:${FMT_NF}" - while read -r pkg; do - echo -e "${FMT_RED} $pkg${FMT_NF}" - done <"$STATUS_DIR/unexpected-failure" - echo "" - fi - - echo_bold "___________________________________________" - echo "" -} - -function report_status_all() { - report_status - without_colors report_status >"$STATUS_DIR/report" -} - -# -# This script will attempt to update every package. It will continue running -# for all the packages even if update fails for a package. -# -# The steps for updating each package are the following: -# 1) Run "buildpkg.sh -u " to update upstream, then attempt merge and -# build. If everything succeeded buildpkg.sh will return success. If only -# some parts succeeded, buildpkg.sh will record status in the package's -# WORKDIR. -# 2) If upstream was updated, push the update to the package repository. -# 3) If a merge with upstream was necessary and we succesfully performed it, -# push the merge to the package repository. -# - -if [[ -n "$UPDATE_PACKAGE_NAME" ]]; then - echo_bold "Updating only package '$UPDATE_PACKAGE_NAME' as" \ - "UPDATE_PACKAGE_NAME is set" - logmust check_package_exists "$UPDATE_PACKAGE_NAME" - PACKAGES=("$UPDATE_PACKAGE_NAME") - echo_bold "auto-mege-blacklist.pkgs is ignored" - NO_MERGE_PACKAGES=() -else - logmust read_package_list "$pkg_list_file" - PACKAGES=("${_RET_LIST[@]}") - logmust read_package_list "$TOP/package-lists/auto-merge-blacklist.pkgs" - NO_MERGE_PACKAGES=("${_RET_LIST[@]}") -fi - -for pkg in "${PACKAGES[@]}"; do - echo "" - echo_bold "Updating package $pkg." - - logmust load_package_config "$pkg" - - WORKDIR="$TOP/packages/$pkg/tmp" - unexpected_failure=false - - flags="" - for no_merge_pkg in "${NO_MERGE_PACKAGES[@]}"; do - if [[ "$pkg" == "$no_merge_pkg" ]]; then - echo_bold "Auto-merge disabled for $pkg as package" \ - "is in auto-merge-blacklist.pkgs" - flags="-M" - fi - done - - echo "Running: ./buildpkg.sh $flags -u $pkg" - if ./buildpkg.sh -u $flags "$pkg"; then - buildpkg_success=true - else - if [[ -f "$WORKDIR/updating-upstream" ]]; then - echo_error "Failed to update upstream for $pkg" - elif [[ -f "$WORKDIR/merging" ]]; then - echo_error "Failed merge with upstream for $pkg" - elif [[ -f "$WORKDIR/building" ]]; then - echo_error "Failed build of $pkg after merge" - else - die "Unexpected failure of buildpkg.sh for $pkg" - fi - - echo_error "buildpkg.sh failed." - echo "$pkg" >>"$STATUS_DIR/update-failed" - buildpkg_success=false - record_failure - fi - - if [[ -f "$WORKDIR/upstream-updated" ]]; then - # - # We push to upstream if it was updated, even if the merge with - # our repo or the build has failed. This way developers can - # manually perform the merge without having to update upstream - # themselves. - # - $dry_run && flags="-n" || flags="" - echo "Running: ./push-updates.sh -u -y $flags $pkg" - if ./push-updates.sh -u -y $flags "$pkg"; then - echo "$pkg" >>"$STATUS_DIR/upstream-pushed" - else - echo_error "Failed to push upstream changes for $pkg." - unexpected_failure=true - record_failure - fi - fi - - if ! $unexpected_failure && [[ -f "$WORKDIR/repo-updated" ]]; then - # sanity check - $buildpkg_success || - die "Repo should not be updated when buildpkg.sh failed" - - $dry_run && flags="-n" || flags="" - echo "Running ./push-updates.sh -m -y $flags $pkg" - if ./push-updates.sh -m -y $flags "$pkg"; then - echo "$pkg" >>"$STATUS_DIR/merge-pushed" - else - echo_error "Failed to push merge for $pkg." - unexpected_failure=true - record_failure - fi - fi - - if $unexpected_failure; then - echo "$pkg" >>"$STATUS_DIR/unexpected-failure" - fi - echo_bold "====================================================================" -done - -trap - EXIT - -report_status_all - -$FAILURE && exit 1 || exit 0 From a47bc3475f2c4b16bb19e433dcc5395efde41b05 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Mon, 2 Nov 2020 13:54:38 -0500 Subject: [PATCH 36/39] Download linux-cloud-tools package when fetching kernel packages (#126) --- lib/common.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/common.sh b/lib/common.sh index f181ea2..26fc17d 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -989,6 +989,15 @@ function fetch_kernel_from_apt_for_version() { "linux-headers-${kernel_version}" \ "linux-tools-${kernel_version}" + # + # For the azure kernel, we also want to get the linux-cloud-tools + # package. Not that we cannot do this indiscriminately since some + # kernel flavors do not come with a linux-cloud-tools package. + # + if [[ "$kernel_version" == *azure ]]; then + logmust apt-get download "linux-cloud-tools-${kernel_version}" + fi + # # Fetch direct dependencies of the downloaded debs. Some of those # dependencies have a slightly different naming scheme than the other From 077b4e0211db25c83a701fcf711bdc084e8db282 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 3 Nov 2020 16:48:59 -0500 Subject: [PATCH 37/39] Fetch dependency of linux dbgsym package when downloading kernel packages (#125) --- lib/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common.sh b/lib/common.sh index 26fc17d..dfa364d 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1004,12 +1004,12 @@ function fetch_kernel_from_apt_for_version() { # kernel packages. # local deb dep deps - for deb in *.deb; do + for deb in *deb; do deps=$(dpkg-deb -f "$deb" Depends | tr -d ' ' | tr ',' ' ') || die "failed to get dependencies for $deb" for dep in $deps; do case "$dep" in - *-headers-* | *-tools-*) + *-headers-* | *-tools-* | *-dbgsym) logmust apt-get download "$dep" ;; esac From 1ca68dc5133356db888722de11532ee53cb74c2b Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 13 Nov 2020 10:17:42 -0500 Subject: [PATCH 38/39] TOOL-10581 linux-pkg should be able to use git token to push to remote (#127) --- lib/common.sh | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/common.sh b/lib/common.sh index dfa364d..0feba5a 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -857,19 +857,17 @@ function merge_with_upstream_default() { # provided in env. # function check_git_credentials_set() { - if [[ -z "$PUSH_GIT_USER" ]] || [[ -z "$PUSH_GIT_PASSWORD" ]]; then + if [[ -z "$PUSH_GIT_TOKEN" ]] && [[ -z "$PUSH_GIT_USER" || -z "$PUSH_GIT_PASSWORD" ]]; then if [[ -t 1 ]]; then if [[ "$DRYRUN" == "false" ]]; then echo_bold "WARNING: this is NOT a dry-run, you are pushing to" \ "a production branch" fi - echo "Please enter git credentials to push to remote ($DEFAULT_PACKAGE_GIT_URL)." - read -r -p "Username: " PUSH_GIT_USER - read -r -s -p "Password: " PUSH_GIT_PASSWORD - export PUSH_GIT_USER - export PUSH_GIT_PASSWORD + echo "Please enter git token to push to remote ($DEFAULT_PACKAGE_GIT_URL)." + read -r -s -p "Token: " PUSH_GIT_TOKEN + export PUSH_GIT_TOKEN else - die "PUSH_GIT_USER and PUSH_GIT_PASSWORD must be set." + die "PUSH_GIT_TOKEN or PUSH_GIT_USER & PUSH_GIT_PASSWORD must be set." fi fi } @@ -888,9 +886,20 @@ function push_to_remote() { logmust check_git_credentials_set - check_env DEFAULT_PACKAGE_GIT_URL PUSH_GIT_USER PUSH_GIT_PASSWORD - local git_url_with_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:${PUSH_GIT_PASSWORD}@}" - local git_url_with_fake_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:@}" + check_env DEFAULT_PACKAGE_GIT_URL + + local git_url_with_creds + local git_url_with_fake_creds + + if [[ -n "$PUSH_GIT_TOKEN" ]]; then + git_url_with_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_TOKEN}@}" + git_url_with_fake_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_TOKEN:0:4}******@}" + elif [[ -n "$PUSH_GIT_USER" && -n "$PUSH_GIT_PASSWORD" ]]; then + git_url_with_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:${PUSH_GIT_PASSWORD}@}" + git_url_with_fake_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:******@}" + else + die "Either PUSH_GIT_TOKEN or PUSH_GIT_USER & PUSH_GIT_PASSWORD must be provided" + fi logmust cd "$WORKDIR/repo" check_git_ref "$local_ref" From 39a3410ee7ba9ccd4edcfeace6b5b86c814ae5f2 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 15 Dec 2020 10:22:12 -0500 Subject: [PATCH 39/39] [Backport of DLPX-73310 to 6.0.6.0] linux-pkg: enable building from release tag (#131) When using a release tag in DEFAULT_GIT_BRANCH the linux-package-mirror link and the base url for fetching build dependencies are now properly set up. --- README.md | 12 +++++++- lib/common.sh | 83 +++++++++++++++++++++++++++++++++++++++++++-------- setup.sh | 14 +++++---- 3 files changed, 90 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ec98f80..116552b 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,9 @@ of some of the scripts defined above. * **DEFAULT_GIT_BRANCH**: The product branch that is being built or updated is typically stored in the file `branch.config`, however it can be overridden via - DEFAULT_GIT_BRANCH. The product branch is used in multiple instances. When + DEFAULT_GIT_BRANCH. It can either be set to a development branch, such as + "master" or "6.0/stage", or a release tag, such as "release/6.0.6.0". + The product branch is used in multiple instances. When running [setup.sh](#setupsh), it will determine what linux-package-mirror link to use when fetching packages from apt (although those links can be overridden via DELPHIX_PACKAGE_MIRROR_MAIN and @@ -309,6 +311,14 @@ of some of the scripts defined above. dependencies produced by the developer Jenkins instance instead of the production one. +* **DEPENDENCIES_BASE_URL**: When fetching artifacts from other linux-pkg + packages that are marked as dependencies of a package, we look for a + specific s3 path based on what product branch or version we are building for, + which is defined by DEFAULT_GIT_BRANCH. If DEPENDENCIES_BASE_URL is left + unset, then the path will be determined automatically. DEPENDENCIES_BASE_URL + is most useful when set to the input-artifacts of a previous appliance-build + run, i.e. "s3://.../input-artifacts/combined-packages/packages". + ## Package Definition For each package built by this framework, there must be a file named diff --git a/lib/common.sh b/lib/common.sh index b946580..daa64a1 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -26,7 +26,6 @@ export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle" # for testing purposes to use jenkins-ops. instead. # export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}" -export _BASE_S3_URL="s3://snapshot-de-images/builds/${JENKINS_OPS_DIR}/devops-gate/master" export UBUNTU_DISTRIBUTION="bionic" @@ -144,6 +143,11 @@ function determine_default_git_branch() { export DEFAULT_GIT_BRANCH } +function is_release_branch() { + check_env DEFAULT_GIT_BRANCH + [[ "$DEFAULT_GIT_BRANCH" == release/* ]] +} + function check_package_exists() { local pkg="$1" @@ -608,6 +612,69 @@ function default_revision() { echo "delphix-$(date '+%Y.%m.%d.%H')" } +function determine_dependencies_base_url() { + [[ -n "$DEPENDENCIES_BASE_URL" ]] && return + + check_env DEFAULT_GIT_BRANCH + + # + # The location of package artifacts depends on whether a release tag + # or a development branch is being built. + # + if is_release_branch; then + local version="${DEFAULT_GIT_BRANCH#release/}" + local url="s3://release-de-images/internal-artifacts/$version" + local suv + suv="$(aws s3 ls "$url/" | awk '{print $2}' | tr -d '/' | sort -V | tail -n 1)" + [[ -n "$suv" ]] || die "No artifacts found at $url" + DEPENDENCIES_BASE_URL="$url/$suv/input-artifacts/combined-packages/packages" + else + DEPENDENCIES_BASE_URL="s3://snapshot-de-images/builds/$JENKINS_OPS_DIR/devops-gate/master/linux-pkg/$DEFAULT_GIT_BRANCH/build-package" + fi + + # + # Make sure the dependencies directory actually exists + # + logmust aws s3 ls "$DEPENDENCIES_BASE_URL" + + export DEPENDENCIES_BASE_URL +} + +function get_package_dependency_s3_url() { + local dep="$1" + + logmust determine_dependencies_base_url + + # + # The base url can be either pointing to: + # 1. Input artifacts from a previously built Delphix Appliance, or + # 2. A build-package/ directory that stores many builds of the + # same package and a "latest" link that points to the latest build + # for that package. + # In case 1 the artifacts are right there under the base url, whereas + # in case 2 we first need to dereference the "latest" link. + # + # The user can provide their own DEPENDENCIES_BASE_URL based on a + # previous Delphix Appliance build or have it determined automatically + # by determine_dependencies_base_url() based on the product branch + # being built. + # + if [[ "$DEPENDENCIES_BASE_URL" == */build-package ]]; then + local s3url="$DEPENDENCIES_BASE_URL/$dep/post-push" + (logmust aws s3 cp --only-show-errors "$s3url/latest" .) || + die "Artifacts for dependency '$dep' missing." \ + "Dependency must be built first." + logmust cat latest + local bucket="${DEPENDENCIES_BASE_URL#s3://}" + bucket=${bucket%%/*} + s3url="s3://$bucket/$(cat latest)" + logmust rm latest + _RET="$s3url" + else + _RET="$DEPENDENCIES_BASE_URL/$dep" + fi +} + # # Fetch artifacts from S3 for all packages listed in PACKAGE_DEPENDENCIES which # is defined in the package's config. @@ -622,11 +689,6 @@ function fetch_dependencies() { return fi - local base_url="$_BASE_S3_URL/linux-pkg/$DEFAULT_GIT_BRANCH/build-package" - - local bucket="${_BASE_S3_URL#s3://}" - bucket=${bucket%%/*} - local dep s3urlvar s3url for dep in $PACKAGE_DEPENDENCIES; do echo "Fetching artifacts for dependency '$dep' ..." @@ -638,13 +700,8 @@ function fetch_dependencies() { "externally" echo "$s3urlvar=$s3url" else - s3url="$base_url/$dep/post-push" - (logmust aws s3 cp --only-show-errors "$s3url/latest" .) || - die "Artifacts for dependency '$dep' missing." \ - "Dependency must be built first." - logmust cat latest - s3url="s3://$bucket/$(cat latest)" - logmust rm latest + logmust get_package_dependency_s3_url "$dep" + s3url="$_RET" fi [[ "$s3url" != */ ]] && s3url="$s3url/" logmust mkdir "$dep" diff --git a/setup.sh b/setup.sh index 2d0b232..0319263 100755 --- a/setup.sh +++ b/setup.sh @@ -31,11 +31,15 @@ configure_apt_sources() { if [[ -z "$primary_url" ]] || [[ -z "$secondary_url" ]]; then local latest_url="http://linux-package-mirror.delphix.com/" - latest_url+="${DEFAULT_GIT_BRANCH}/latest/" - package_mirror_url=$(curl -LfSs -o /dev/null -w '%{url_effective}' \ - "$latest_url" || die "Could not curl $latest_url") - # Remove trailing slash, if present. - package_mirror_url="${package_mirror_url%/}" + if is_release_branch; then + package_mirror_url="${latest_url}${DEFAULT_GIT_BRANCH}" + else + latest_url+="${DEFAULT_GIT_BRANCH}/latest/" + package_mirror_url=$(curl -LfSs -o /dev/null -w '%{url_effective}' \ + "$latest_url" || die "Could not curl $latest_url") + # Remove trailing slash, if present. + package_mirror_url="${package_mirror_url%/}" + fi [[ -z "$primary_url" ]] && primary_url="${package_mirror_url}/ubuntu" [[ -z "$secondary_url" ]] && secondary_url="${package_mirror_url}/ppas" fi