From 22b3e970bcbf7871e6fdf2d5d2974e435f95df7d Mon Sep 17 00:00:00 2001 From: Palash Gandhi Date: Thu, 3 Oct 2024 12:13:41 -0700 Subject: [PATCH] Stop relying on internal sources in favor of upstream packages for LTS upgrade PR URL: https://www.github.com/delphix/linux-pkg/pull/325 --- .github/scripts/verify-query-packages.sh | 8 --- README.md | 7 +- package-lists/build/main.pkgs | 8 --- packages/adoptopenjdk/config.sh | 81 ------------------------ packages/bcc/config.sh | 39 ------------ packages/bpftrace/config.sh | 42 ------------ packages/containerized-masking/config.sh | 8 +-- packages/delphix-sso-app/config.sh | 8 +-- packages/drgn/config.sh | 41 ------------ packages/fio/config.sh | 41 ------------ packages/gdb-python/config.sh | 27 -------- packages/libkdumpfile/config.sh | 34 ---------- packages/masking/config.sh | 8 +-- packages/mold/config.sh | 34 ---------- packages/virtualization/config.sh | 6 +- 15 files changed, 7 insertions(+), 385 deletions(-) delete mode 100755 packages/adoptopenjdk/config.sh delete mode 100644 packages/bcc/config.sh delete mode 100644 packages/bpftrace/config.sh delete mode 100644 packages/drgn/config.sh delete mode 100644 packages/fio/config.sh delete mode 100644 packages/gdb-python/config.sh delete mode 100644 packages/libkdumpfile/config.sh delete mode 100644 packages/mold/config.sh diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index 65cc85f..c405117 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -30,14 +30,6 @@ read -r -a fields <<<"$(./query-packages.sh single -o git-url zfs 2>&1)" test ${#fields[@]} -eq 1 test "${fields[0]}" == 'https://github.com/delphix/zfs.git' -# Expect: "bpftrace bcc true https://github.com/delphix/bpftrace.git" -read -r -a fields <<<"$(./query-packages.sh single -o name,dependencies,can-update,git-url bpftrace 2>&1)" -test ${#fields[@]} -eq 4 -test "${fields[0]}" == 'bpftrace' -test "${fields[1]}" == 'bcc' -test "${fields[2]}" == 'true' -test "${fields[3]}" == 'https://github.com/delphix/bpftrace.git' - # Expect that "list all" outputs all directory names under packages/ diff <(ls -1 packages | sort) <(./query-packages.sh list all 2>&1 | sort) diff --git a/README.md b/README.md index ae292b4..4a5aa0b 100644 --- a/README.md +++ b/README.md @@ -280,12 +280,7 @@ of some of the scripts defined above. * **package_S3_URL**: Similar to the package_VAR variables above. This is used to override the default S3 location for where package build-dependencies are - fetched for a given linux-pkg package. For instance, if you are building - bpftrace, which has `PACKAGE_DEPENDENCIES="bcc"` in its config, the - `fetch_dependencies()` stage in the build will fetch the latest build - artifacts of the bcc package from a predetermined S3 location. If you pass - `BCC_S3_URL=s3://path/to/custom/bcc/artifacts` then those artifacts will be - fetched insteasd. + fetched for a given linux-pkg package. * **DELPHIX_PACKAGE_MIRROR_MAIN, DELPHIX_PACKAGE_MIRROR_SECONDARY**: When the [setup.sh](#setupsh) script is run, it will configure the apt sources diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs index d96c1fb..85dfb82 100644 --- a/package-lists/build/main.pkgs +++ b/package-lists/build/main.pkgs @@ -2,9 +2,6 @@ # List of non-kernel packages to be included in the Delphix Appliance. # -adoptopenjdk -bcc -bpftrace challenge-response cloud-init crash-python @@ -13,20 +10,15 @@ delphix-go delphix-platform delphix-rust delphix-sso-app -drgn docker-python-image dwarves -fio fluentd-gems -gdb-python grub2 host-jdks -libkdumpfile make-jpkg makedumpfile masking misc-debs -mold nfs-utils performance-diagnostics ptools diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh deleted file mode 100755 index 8342afd..0000000 --- a/packages/adoptopenjdk/config.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2024 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL=none -PACKAGE_DEPENDENCIES="make-jpkg" - -case $(dpkg-architecture -q DEB_HOST_ARCH 2>/dev/null || echo "none") in -amd64) - _tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u422b05.tar.gz" - _tarfile_sha256="4c6056f6167fae73ace7c3080b78940be5c87d54f5b08894b3517eed1cbb2c06" - _jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" - ;; -arm64) - _tarfile="OpenJDK8U-jdk_aarch64_linux_hotspot_8u422b05.tar.gz" - _tarfile_sha256="af98a839ec238106078bd360af9e405dc6665c05ee837178ed13b92193681923" - _jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-arm64" - ;; -*) ;; - -esac - -function prepare() { - logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb -} - -function fetch() { - # We exit here rather than above in the architecture detection logic - # to deal with the fact that this file can, during test runs, be - # sourced on platforms where builds are not happening. list-packages - # sources the file to gather information about the package, and this - # is performed on jenkins and macos during test runs. Having the exit - # occur above causes those runs to fail. - if [[ -z "$_tarfile" ]]; then - echo "Invalid architecture detected" >&2 - exit 1 - fi - logmust cd "$WORKDIR/" - - local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile" - - logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256" -} - -function build() { - if [[ -z "$_tarfile" ]]; then - echo "Invalid architecture detected" >&2 - exit 1 - fi - logmust cd "$WORKDIR/" - - logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<'$WORKDIR/artifacts/JDK_PATH'" - - echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO" -} diff --git a/packages/bcc/config.sh b/packages/bcc/config.sh deleted file mode 100644 index 17ac503..0000000 --- a/packages/bcc/config.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bcc.git" - -UPSTREAM_GIT_URL=https://github.com/iovisor/bcc.git -UPSTREAM_GIT_BRANCH=master - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust cd "$WORKDIR/repo" - # Note: the string to determine the version was copied from bcc's - # debian/rules file. - PACKAGE_VERSION=$(dpkg-parsechangelog | sed -rne 's,^Version: (.*),\1,p') - - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/bpftrace/config.sh b/packages/bpftrace/config.sh deleted file mode 100644 index 79cfa30..0000000 --- a/packages/bpftrace/config.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bpftrace.git" -PACKAGE_DEPENDENCIES="bcc" - -UPSTREAM_GIT_URL="https://github.com/iovisor/bpftrace.git" -UPSTREAM_GIT_BRANCH="master" - -function fetch() { - logmust fetch_repo_from_git - logmust git submodule init - logmust git submodule update --recursive -} - -function prepare() { - logmust install_pkgs "$DEPDIR"/bcc/libbcc_*.deb - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index 7c480b5..12cf1d8 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -29,23 +29,19 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" SKIP_COPYRIGHTS_CHECK=true function prepare() { - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb + echo "Nothing to prepare" } function build() { - export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" - logmust cd "$WORKDIR/repo" local args=() set_secret_build_args + args+=("-Dorg.gradle.java.home=/lib/jvm/java-8-openjdk-amd64/") args+=("${_SECRET_BUILD_ARGS[@]}") args+=("-Porg.gradle.configureondemand=false") diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 3f88014..10bced2 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -17,17 +17,13 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/saml-app.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb + echo "Nothing to prepare" } function build() { - local java_home - java_home=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" logmust cd "$WORKDIR/repo" - logmust sudo ./gradlew "-Dorg.gradle.java.home=$java_home" distDeb + logmust sudo ./gradlew distDeb logmust sudo mv ./build/distributions/*deb "$WORKDIR/artifacts/" } diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh deleted file mode 100644 index 0387a3a..0000000 --- a/packages/drgn/config.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/drgn.git" -PACKAGE_DEPENDENCIES="libkdumpfile" - -UPSTREAM_GIT_URL="https://github.com/osandov/drgn.git" -UPSTREAM_GIT_BRANCH="main" - -function prepare() { - # - # Strictly speaking libkdumpfile is not a hard prerequisite for - # drgn itself, but it is a hard requirement in our use-case as - # we do want to use drgn for kdump-compressed crash dumps. - # - logmust install_pkgs "$DEPDIR"/libkdumpfile/*.deb - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/fio/config.sh b/packages/fio/config.sh deleted file mode 100644 index 758ff25..0000000 --- a/packages/fio/config.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018, 2022 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/fio.git" - -function prepare() { - logmust install_pkgs \ - libaio-dev \ - librdmacm-dev \ - libibverbs-dev \ - librbd-dev \ - libgtk2.0-dev \ - libcairo2-dev \ - libnuma-dev \ - flex \ - bison \ - libglusterfs-dev \ - libpmem-dev \ - libpmemblk-dev -} - -function build() { - logmust cd "$WORKDIR/repo/" - logmust dpkg-buildpackage -b -us -uc - logmust mv "$WORKDIR"/*deb "$WORKDIR/artifacts/" -} diff --git a/packages/gdb-python/config.sh b/packages/gdb-python/config.sh deleted file mode 100644 index df5e2be..0000000 --- a/packages/gdb-python/config.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/gdb-python.git" - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} diff --git a/packages/libkdumpfile/config.sh b/packages/libkdumpfile/config.sh deleted file mode 100644 index f6e5c17..0000000 --- a/packages/libkdumpfile/config.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019, 2020 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# shellcheck disable=SC2034 -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/libkdumpfile.git" - -UPSTREAM_GIT_URL="https://github.com/ptesarik/libkdumpfile.git" -UPSTREAM_GIT_BRANCH="tip" - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 2a641c7..71c0b22 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -19,20 +19,13 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk" function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" - - logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb } function build() { - export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" - logmust cd "$WORKDIR/repo" # @@ -51,6 +44,7 @@ function build() { local args=() set_secret_build_args + args+=("-Dorg.gradle.java.home=/lib/jvm/java-8-openjdk-amd64/") args+=("${_SECRET_BUILD_ARGS[@]}") args+=("-Porg.gradle.configureondemand=false") diff --git a/packages/mold/config.sh b/packages/mold/config.sh deleted file mode 100644 index 589ef63..0000000 --- a/packages/mold/config.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2023 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/mold.git" - -UPSTREAM_GIT_URL=https://salsa.debian.org/pkg-llvm-team/mold.git -UPSTREAM_GIT_BRANCH=master - -function prepare() { - logmust install_build_deps_from_control_file -} - -function build() { - logmust dpkg_buildpackage_default -} - -function update_upstream() { - logmust update_upstream_from_git -} diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index 1f72783..c8b4c82 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -19,22 +19,18 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" -PACKAGE_DEPENDENCIES="adoptopenjdk crypt-blowfish host-jdks" +PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" function prepare() { logmust read_list "$WORKDIR/repo/appliance/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" logmust install_pkgs \ - "$DEPDIR"/adoptopenjdk/*.deb \ "$DEPDIR"/crypt-blowfish/*.deb \ "$DEPDIR"/host-jdks/*.deb } function build() { - export JAVA_HOME - JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || - die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" export LANG LANG=en_US.UTF-8