diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index 65cc85f0..c4051171 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -30,14 +30,6 @@ read -r -a fields <<<"$(./query-packages.sh single -o git-url zfs 2>&1)" test ${#fields[@]} -eq 1 test "${fields[0]}" == 'https://github.com/delphix/zfs.git' -# Expect: "bpftrace bcc true https://github.com/delphix/bpftrace.git" -read -r -a fields <<<"$(./query-packages.sh single -o name,dependencies,can-update,git-url bpftrace 2>&1)" -test ${#fields[@]} -eq 4 -test "${fields[0]}" == 'bpftrace' -test "${fields[1]}" == 'bcc' -test "${fields[2]}" == 'true' -test "${fields[3]}" == 'https://github.com/delphix/bpftrace.git' - # Expect that "list all" outputs all directory names under packages/ diff <(ls -1 packages | sort) <(./query-packages.sh list all 2>&1 | sort) diff --git a/README.md b/README.md index ae292b4b..4a5aa0b4 100644 --- a/README.md +++ b/README.md @@ -280,12 +280,7 @@ of some of the scripts defined above. * **package_S3_URL**: Similar to the package_VAR variables above. This is used to override the default S3 location for where package build-dependencies are - fetched for a given linux-pkg package. For instance, if you are building - bpftrace, which has `PACKAGE_DEPENDENCIES="bcc"` in its config, the - `fetch_dependencies()` stage in the build will fetch the latest build - artifacts of the bcc package from a predetermined S3 location. If you pass - `BCC_S3_URL=s3://path/to/custom/bcc/artifacts` then those artifacts will be - fetched insteasd. + fetched for a given linux-pkg package. * **DELPHIX_PACKAGE_MIRROR_MAIN, DELPHIX_PACKAGE_MIRROR_SECONDARY**: When the [setup.sh](#setupsh) script is run, it will configure the apt sources diff --git a/default-package-config.sh b/default-package-config.sh index 1a721d4e..13775d03 100644 --- a/default-package-config.sh +++ b/default-package-config.sh @@ -53,8 +53,8 @@ function kernel_build() { local platform="$1" # # Note: Extra arguments can overwrite default arguments. - # For example in this function we default skipdbg - # to false, but if we pass "skipdbg=true" as an + # For example in this function we default do_dbgsym_package + # to false, but if we pass "do_dbgsym_package=true" as an # extra argument we will be overwriting this value # to true. This is because when a variable's value # is declared multiple times when invoking the @@ -103,25 +103,26 @@ function kernel_build() { echo "$kernel_version" >"$WORKDIR/artifacts/KERNEL_VERSION" # - # skipdbg=false + # do_dbgsym_package=true # We need debug info for our debugging tools to work. # Don't skip them. + # do_tools_common=false + # We do not need to build linux-tools-common package and we + # install it directly from our package mirror. # uefi_signed=false # This variable defaults to true but since we don't have # any intention and logic to provide signatures for now # we set it to false to avoid any misconfigurations down # the line. - # disable_d_i=true - # This prevents udeb packages from being built as they are - # not consumed by the Delphix Appliance. # do_dkms_*=false # This disables the build of various out-of-tree kernel modules # that we do not use in our product or that we provide separately. # local debian_rules_args=( - "skipdbg=false" + "do_dbgsym_package=true" + "do_tools_common=false" + "do_tools_host=false" "uefi_signed=false" - "disable_d_i=true" "do_zfs=false" "do_dkms_nvidia=false" "do_dkms_nvidia_server=false" diff --git a/lib/common.sh b/lib/common.sh index d81dded2..1064abcd 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -616,6 +616,10 @@ function delphix_revision() { echo "delphix.$(date '+%Y.%m.%d.%H.%M')" } +function compare_versions() { + dpkg --compare-versions "$@" +} + function determine_dependencies_base_url() { [[ -n "$DEPENDENCIES_BASE_URL" ]] && return diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs index d96c1fb9..85dfb828 100644 --- a/package-lists/build/main.pkgs +++ b/package-lists/build/main.pkgs @@ -2,9 +2,6 @@ # List of non-kernel packages to be included in the Delphix Appliance. # -adoptopenjdk -bcc -bpftrace challenge-response cloud-init crash-python @@ -13,20 +10,15 @@ delphix-go delphix-platform delphix-rust delphix-sso-app -drgn docker-python-image dwarves -fio fluentd-gems -gdb-python grub2 host-jdks -libkdumpfile make-jpkg makedumpfile masking misc-debs -mold nfs-utils performance-diagnostics ptools diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh deleted file mode 100755 index 8342afd9..00000000 --- a/packages/adoptopenjdk/config.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2024 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL=none -PACKAGE_DEPENDENCIES="make-jpkg" - -case $(dpkg-architecture -q DEB_HOST_ARCH 2>/dev/null || echo "none") in -amd64) - _tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u422b05.tar.gz" - _tarfile_sha256="4c6056f6167fae73ace7c3080b78940be5c87d54f5b08894b3517eed1cbb2c06" - _jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" - ;; -arm64) - _tarfile="OpenJDK8U-jdk_aarch64_linux_hotspot_8u422b05.tar.gz" - _tarfile_sha256="af98a839ec238106078bd360af9e405dc6665c05ee837178ed13b92193681923" - _jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-arm64" - ;; -*) ;; - -esac - -function prepare() { - logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb -} - -function fetch() { - # We exit here rather than above in the architecture detection logic - # to deal with the fact that this file can, during test runs, be - # sourced on platforms where builds are not happening. list-packages - # sources the file to gather information about the package, and this - # is performed on jenkins and macos during test runs. Having the exit - # occur above causes those runs to fail. - if [[ -z "$_tarfile" ]]; then - echo "Invalid architecture detected" >&2 - exit 1 - fi - logmust cd "$WORKDIR/" - - local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile" - - logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256" -} - -function build() { - if [[ -z "$_tarfile" ]]; then - echo "Invalid architecture detected" >&2 - exit 1 - fi - logmust cd "$WORKDIR/" - - logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<'$WORKDIR/artifacts/JDK_PATH'" - - echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO" -} diff --git a/packages/bcc/config.sh b/packages/bcc/config.sh deleted file mode 100644 index 17ac5032..00000000 --- a/packages/bcc/config.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bcc.git" - -UPSTREAM_GIT_URL=https://github.com/iovisor/bcc.git -UPSTREAM_GIT_BRANCH=master - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust cd "$WORKDIR/repo" - # Note: the string to determine the version was copied from bcc's - # debian/rules file. - PACKAGE_VERSION=$(dpkg-parsechangelog | sed -rne 's,^Version: (.*),\1,p') - - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/bpftrace/config.sh b/packages/bpftrace/config.sh deleted file mode 100644 index 79cfa307..00000000 --- a/packages/bpftrace/config.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bpftrace.git" -PACKAGE_DEPENDENCIES="bcc" - -UPSTREAM_GIT_URL="https://github.com/iovisor/bpftrace.git" -UPSTREAM_GIT_BRANCH="master" - -function fetch() { - logmust fetch_repo_from_git - logmust git submodule init - logmust git submodule update --recursive -} - -function prepare() { - logmust install_pkgs "$DEPDIR"/bcc/libbcc_*.deb - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/connstat/config.sh b/packages/connstat/config.sh index 33825120..1c8559a3 100644 --- a/packages/connstat/config.sh +++ b/packages/connstat/config.sh @@ -23,7 +23,7 @@ function prepare() { logmust install_pkgs \ debhelper \ dpkg-dev \ - llvm-12 + llvm-14 logmust install_kernel_headers logmust install_pkgs "$DEPDIR"/dwarves/*.deb } diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index 7c480b59..7b2fd926 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -29,17 +29,15 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" SKIP_COPYRIGHTS_CHECK=true function prepare() { - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb + echo "Nothing to prepare" } function build() { export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" logmust cd "$WORKDIR/repo" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 3f880145..79ac9139 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -17,16 +17,14 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/saml-app.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb + echo "Nothing to prepare" } function build() { local java_home - java_home=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + java_home="/usr/lib/jvm/java-8-openjdk-amd64/" logmust cd "$WORKDIR/repo" logmust sudo ./gradlew "-Dorg.gradle.java.home=$java_home" distDeb logmust sudo mv ./build/distributions/*deb "$WORKDIR/artifacts/" diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh deleted file mode 100644 index 0387a3a7..00000000 --- a/packages/drgn/config.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/drgn.git" -PACKAGE_DEPENDENCIES="libkdumpfile" - -UPSTREAM_GIT_URL="https://github.com/osandov/drgn.git" -UPSTREAM_GIT_BRANCH="main" - -function prepare() { - # - # Strictly speaking libkdumpfile is not a hard prerequisite for - # drgn itself, but it is a hard requirement in our use-case as - # we do want to use drgn for kdump-compressed crash dumps. - # - logmust install_pkgs "$DEPDIR"/libkdumpfile/*.deb - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/fio/config.sh b/packages/fio/config.sh deleted file mode 100644 index 758ff252..00000000 --- a/packages/fio/config.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2022 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/fio.git" - -function prepare() { - logmust install_pkgs \ - libaio-dev \ - librdmacm-dev \ - libibverbs-dev \ - librbd-dev \ - libgtk2.0-dev \ - libcairo2-dev \ - libnuma-dev \ - flex \ - bison \ - libglusterfs-dev \ - libpmem-dev \ - libpmemblk-dev -} - -function build() { - logmust cd "$WORKDIR/repo/" - logmust dpkg-buildpackage -b -us -uc - logmust mv "$WORKDIR"/*deb "$WORKDIR/artifacts/" -} diff --git a/packages/gdb-python/config.sh b/packages/gdb-python/config.sh deleted file mode 100644 index df5e2beb..00000000 --- a/packages/gdb-python/config.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/gdb-python.git" - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} diff --git a/packages/libkdumpfile/config.sh b/packages/libkdumpfile/config.sh deleted file mode 100644 index f6e5c179..00000000 --- a/packages/libkdumpfile/config.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/libkdumpfile.git" - -UPSTREAM_GIT_URL="https://github.com/ptesarik/libkdumpfile.git" -UPSTREAM_GIT_BRANCH="tip" - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 2a641c7f..963a6c99 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -19,19 +19,15 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" - - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb } function build() { export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" logmust cd "$WORKDIR/repo" diff --git a/packages/mold/config.sh b/packages/mold/config.sh deleted file mode 100644 index 589ef63a..00000000 --- a/packages/mold/config.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2023 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/mold.git" - -UPSTREAM_GIT_URL=https://salsa.debian.org/pkg-llvm-team/mold.git -UPSTREAM_GIT_BRANCH=master - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index 1f72783f..e4b015c7 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -19,22 +19,20 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk crypt-blowfish host-jdks" +PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" function prepare() { logmust read_list "$WORKDIR/repo/appliance/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" logmust install_pkgs \ - "$DEPDIR"/adoptopenjdk/*.deb \ "$DEPDIR"/crypt-blowfish/*.deb \ "$DEPDIR"/host-jdks/*.deb } function build() { export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" export LANG LANG=en_US.UTF-8 diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index 6d9e0aed..94d764a9 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -33,6 +33,7 @@ function prepare() { devscripts \ dh-autoreconf \ dh-python \ + dh-dkms \ dkms \ fakeroot \ gawk \ @@ -40,12 +41,14 @@ function prepare() { libattr1-dev \ libblkid-dev \ libelf-dev \ + libpam0g \ libselinux-dev \ libselinux1-dev \ libssl-dev \ + libtirpc-dev \ libtool \ libudev-dev \ - llvm-12 \ + llvm-14 \ lsb-release \ lsscsi \ parted \ diff --git a/setup.sh b/setup.sh index 9be7405f..98398b46 100755 --- a/setup.sh +++ b/setup.sh @@ -19,7 +19,7 @@ TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" source "$TOP/lib/common.sh" logmust check_running_system -check_env DEFAULT_GIT_BRANCH +check_env DEFAULT_GIT_BRANCH DELPHIX_RELEASE_VERSION # # Update the sources.list file to point to our internal package mirror. If no @@ -31,7 +31,16 @@ function configure_apt_sources() { local secondary_url="$DELPHIX_PACKAGE_MIRROR_SECONDARY" if [[ -z "$primary_url" ]] || [[ -z "$secondary_url" ]]; then - local latest_url="http://linux-package-mirror.delphix.com/" + local latest_url + local delphix_version="$DELPHIX_RELEASE_VERSION" + # TODO: Remove the first condition & set the version of os-upgrade in dlpx.version.mapping to + # 9999.0.0.0 when https://perforce.atlassian.net/browse/SUP-5179 is done. + if compare_versions "$delphix_version" eq "999.0.0.0" || compare_versions "$delphix_version" gt "2025.2"; then + latest_url="http://linux-package-mirror-v2.delphix.com/" + else + latest_url="http://linux-package-mirror.delphix.com/" + fi + if is_release_branch; then package_mirror_url="${latest_url}releases/${DELPHIX_RELEASE_VERSION}" else @@ -74,7 +83,6 @@ function configure_apt_sources() { deb-src ${primary_url} ${UBUNTU_DISTRIBUTION}-backports main restricted universe multiverse deb ${secondary_url} ${UBUNTU_DISTRIBUTION} main multiverse universe - deb ${secondary_url} ${UBUNTU_DISTRIBUTION}-updates main multiverse universe EOF" || die "/etc/apt/sources.list could not be updated" logmust sudo apt-key add "$TOP/resources/delphix-secondary-mirror.key"