-
Notifications
You must be signed in to change notification settings - Fork 31
/
common.sh
1357 lines (1174 loc) · 38.6 KB
/
common.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
#
# Copyright 2018, 2021 Delphix
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
export _RET
export _RET_LIST
export _SECRET_BUILD_ARGS
export DEBIAN_FRONTEND=noninteractive
export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle"
#
# Used when fetching artifacts for external dependencies. Can be overridden
# for testing purposes to use jenkins-ops.<developer> instead.
#
export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}"
export UBUNTU_DISTRIBUTION="focal"
#
# We currently support getting the linux kernel from 3 different sources:
# 1. "delphix": building it from code
# 2. "archive": dowloading from apt
# 3. "prebuilt": pre-built kernel stored in artifactory
#
export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="delphix"
# shellcheck disable=SC2086
function enable_colors() {
[[ -t 1 ]] && flags="" || flags="-T xterm"
FMT_RED="$(tput $flags setaf 1)"
FMT_GREEN="$(tput $flags setaf 2)"
FMT_BOLD="$(tput $flags bold)"
FMT_NF="$(tput $flags sgr0)"
COLORS_ENABLED=true
}
function disable_colors() {
FMT_RED=""
FMT_GREEN=""
FMT_BOLD=""
FMT_NF=""
COLORS_ENABLED=false
}
if [[ -t 1 ]] || [[ "$FORCE_COLORS" == "true" ]]; then
enable_colors
else
disable_colors
fi
function without_colors() {
if [[ "$COLORS_ENABLED" == "true" ]]; then
disable_colors
"$@"
enable_colors
else
"$@"
fi
}
function echo_error() {
echo -e "${FMT_BOLD}${FMT_RED}Error: $*${FMT_NF}" >&2
}
function echo_success() {
echo -e "${FMT_BOLD}${FMT_GREEN}Success: $*${FMT_NF}"
}
function echo_bold() {
echo -e "${FMT_BOLD}$*${FMT_NF}"
}
function die() {
[[ $# -gt 0 ]] && echo_error "$*"
exit 1
}
function logmust() {
[[ "$LOGGING" == "false" ]] || echo Running: "$@" >&2
"$@" || die "failed command '$*'"
}
#
# Check that we are running in AWS on an Ubuntu system of the appropriate
# distribution. This is not a strict requirement for the build to work but
# rather a safety measure to prevent developers from accidentally running the
# scripts on their work system and changing its configuration.
#
function check_running_system() {
local msg
if [[ "$DISABLE_SYSTEM_CHECK" == "true" ]]; then
echo "WARNING: System check disabled."
return 0
fi
msg="Note that you can bypass this check by setting environment"
msg="${msg} variable DISABLE_SYSTEM_CHECK=true. Use this at your"
msg="${msg} own risk as running this command may modify your system."
if ! (command -v lsb_release >/dev/null &&
[[ $(lsb_release -cs) == "$UBUNTU_DISTRIBUTION" ]]); then
echo_error "Script can only be run on an ubuntu-${UBUNTU_DISTRIBUTION} system."
echo_bold "$msg"
exit 1
fi
if ! curl "http://169.254.169.254/latest/meta-datas" \
>/dev/null 2>&1; then
echo_error "Not running in AWS, are you sure you are on the" \
"right system?"
echo_bold "$msg"
exit 1
fi
}
#
# We need to have run setup.sh before running most linux-pkg commands.
# This checks if setup has been run before. Note that if the system was
# rebooted we want to rerun setup as cloud-init will reset apt sources on
# boot.
#
function run_setup_if_needed() {
[[ -f /run/linux-pkg-setup ]] && return
check_env TOP
echo_bold "------------------------------------------------------------"
echo_bold "Running setup script"
echo_bold "------------------------------------------------------------"
logmust "$TOP/setup.sh"
echo_bold "------------------------------------------------------------"
}
function is_release_branch() {
check_env DEFAULT_GIT_BRANCH
[[ "$DEFAULT_GIT_BRANCH" == release/* ]]
}
function check_package_exists() {
local pkg="$1"
check_env TOP
echo "$pkg" | grep -q '/' && die "Package name should not contain '/'"
[[ -d "$TOP/packages/$pkg" ]] || die "Unknown package '$pkg'."
}
function check_env() {
local var val required
required=true
for var in "$@"; do
if [[ "$var" == "--" ]]; then
required=false
continue
fi
val="${!var}"
if $required && [[ -z "$val" ]]; then
die "check_env: $var must be non-empty"
fi
done
}
function check_git_ref() {
local ref
for ref in "$@"; do
if ! git show-ref -q "$ref"; then
die "git ref '$ref' not found"
fi
done
}
#
# execute a hook from a package's config.sh
#
function stage() {
typeset hook=$1
shift 1
check_env PACKAGE
local stage_start=$SECONDS
echo ""
if type -t "$hook" >/dev/null; then
echo_bold "PACKAGE $PACKAGE: STAGE $hook STARTED"
logmust "$hook" "$@"
echo_bold "PACKAGE $PACKAGE: STAGE $hook COMPLETED in" \
"$((SECONDS - stage_start)) seconds"
else
echo_bold "PACKAGE $PACKAGE: SKIPPING UNDEFINED STAGE $hook"
fi
echo ""
}
function reset_package_config_variables() {
local hook
local var
local hooks="
prepare
fetch
build
checkstyle
update_upstream
merge_with_upstream
"
for hook in $hooks; do
unset "$hook"
done
local vars="
PACKAGE_GIT_URL
PACKAGE_GIT_BRANCH
PACKAGE_VERSION
PACKAGE_REVISION
DEFAULT_PACKAGE_GIT_URL
PACKAGE_DEPENDENCIES
UPSTREAM_SOURCE_PACKAGE
UPSTREAM_GIT_URL
UPSTREAM_GIT_BRANCH
WORKDIR
PKGDIR
PACKAGE_PREFIX
FORCE_PUSH_ON_UPDATE
SKIP_COPYRIGHTS_CHECK
"
for var in $vars; do
unset "$var"
done
}
function get_package_prefix() {
local pkg="$1"
local pkg_prefix
#
# We allow overriding package-specific configuration through
# environment variables starting with <pkg_prefix> in
# get_package_config_from_env(). We make sure that the names of new
# packages can be converted to a valid <pkg_prefix>.
#
pkg_prefix="$(echo "$pkg" | tr - _ | tr '[:lower:]' '[:upper:]')"
[[ "$pkg_prefix" =~ ^[A-Z][A-Z0-9_]*$ ]] ||
die "Failed to convert package name '$pkg' to valid" \
"prefix ($pkg_prefix)"
_RET="$pkg_prefix"
}
#
# Loads configuration for building package passed in $1. High level tasks are:
# 1. Reset/Cleanup package configuration environment
# 2. Source default config for all packages: default-package-config.sh
# 3. Source default config for specific package: packages/PACKAGE/config.sh
# 4. Look at environment variables that can override default configs.
# 5. Validate config
#
function load_package_config() {
export PACKAGE="$1"
logmust check_package_exists "$PACKAGE"
#
# unset hooks and variables that are reserved for a package's config.
#
logmust reset_package_config_variables
check_env TOP
export PKGDIR="$TOP/packages/$PACKAGE"
export WORKDIR="$PKGDIR/tmp"
logmust source "$TOP/default-package-config.sh"
logmust source "$PKGDIR/config.sh"
#
# A package's config.sh file can define default values for:
# GIT_URL, GIT_BRANCH, VERSION, REVISION.
#
# Those defaults can be overriden either by package-specific
# environment variables or by parameters passed from command line.
#
logmust get_package_prefix "$PACKAGE"
export PACKAGE_PREFIX="$_RET"
logmust get_package_config_from_env
#
# Check that package configuration is valid
#
[[ -n "$DEFAULT_PACKAGE_GIT_URL" ]] ||
die "$PACKAGE: DEFAULT_PACKAGE_GIT_URL is not defined. Set " \
"it to 'none' if the source is not fetched from git"
[[ "$DEFAULT_PACKAGE_GIT_URL" == https://* ]] ||
[[ "$DEFAULT_PACKAGE_GIT_URL" == "none" ]] ||
die "$PACKAGE: DEFAULT_PACKAGE_GIT_URL must begin with " \
"https:// or be set to 'none'"
local dependency
local deps_array=()
for dependency in $PACKAGE_DEPENDENCIES; do
#
# Check for special value @linux-kernel which resolves to
# all flavors of linux kernel packages.
#
if [[ $dependency == '@linux-kernel' ]]; then
logmust list_linux_kernel_packages
deps_array+=("${_RET_LIST[@]}")
continue
fi
(check_package_exists "$dependency") ||
die "Invalid package dependency '$dependency'"
deps_array+=("$dependency")
done
PACKAGE_DEPENDENCIES="${deps_array[*]}"
#
# Check for variables related to update_upstream() hook
#
local found=false
if [[ -n "$UPSTREAM_GIT_URL" ]]; then
[[ -n "$UPSTREAM_GIT_BRANCH" ]] ||
die "$PACKAGE: UPSTREAM_GIT_BRANCH must also be" \
"defined when UPSTREAM_GIT_URL is defined."
found=true
elif [[ -n "$UPSTREAM_GIT_BRANCH" ]]; then
die "$PACKAGE: UPSTREAM_GIT_URL must also be defined when" \
"UPSTREAM_GIT_BRANCH is defined."
fi
if [[ -n "$UPSTREAM_SOURCE_PACKAGE" ]]; then
$found && die "$PACKAGE: UPSTREAM_SOURCE_PACKAGE and" \
"UPSTREAM_GIT_URL are mutually exclusive."
found=true
fi
if $found && ! type -t update_upstream >/dev/null; then
die "$PACKAGE: update_upstream() hook must be defined when" \
"either UPSTREAM_SOURCE_PACKAGE or UPSTREAM_GIT_URL" \
"is set."
fi
#
# Check that mandatory hooks are defined
#
for hook in fetch build; do
type -t "$hook" >/dev/null ||
die "$PACKAGE: Hook '$hook' missing."
done
}
#
# Use different config sources to determine the values for:
# PACKAGE_GIT_URL, PACKAGE_GIT_BRANCH, PACKAGE_REVISION
#
# The sources for the config, in decreasing order of priority, are:
# 1. Command line parameters passed to build script.
# 2. Package-specific environment variables {PACKAGE_PREFIX}_{SUFFIX}.
# PACKAGE_PREFIX is the package's name in CAPS with '-' replaced by '_'.
# E.g. CLOUD_INIT_GIT_URL sets PACKAGE_GIT_URL for package cloud-init.
# 3. DEFAULT_PACKAGE_{SUFFIX} variables defined in package's config.sh.
# 4. Global defaults for all packages, DEFAULT_{SUFFIX}.
#
# This function should be called after loading a package's config.sh.
#
function get_package_config_from_env() {
local var
check_env PACKAGE_PREFIX
echo "get_package_config_from_env(): using prefix: ${PACKAGE_PREFIX}_"
var="${PACKAGE_PREFIX}_GIT_URL"
if [[ -n "$PARAM_PACKAGE_GIT_URL" ]]; then
PACKAGE_GIT_URL="$PARAM_PACKAGE_GIT_URL"
echo "PARAM_PACKAGE_GIT_URL passed from '-g'"
elif [[ -n "${!var}" ]]; then
PACKAGE_GIT_URL="${!var}"
echo "PACKAGE_GIT_URL set to value of ${var}"
elif [[ -n "$DEFAULT_PACKAGE_GIT_URL" ]]; then
PACKAGE_GIT_URL="$DEFAULT_PACKAGE_GIT_URL"
echo "PACKAGE_GIT_URL set to value of DEFAULT_PACKAGE_GIT_URL"
fi
var="${PACKAGE_PREFIX}_GIT_BRANCH"
if [[ -n "$PARAM_PACKAGE_GIT_BRANCH" ]]; then
PACKAGE_GIT_BRANCH="$PARAM_PACKAGE_GIT_BRANCH"
echo "PARAM_PACKAGE_GIT_BRANCH passed from '-b'"
elif [[ -n "${!var}" ]]; then
PACKAGE_GIT_BRANCH="${!var}"
echo "PACKAGE_GIT_BRANCH set to value of ${var}"
fi
if [[ -z "$PACKAGE_GIT_BRANCH" ]]; then
PACKAGE_GIT_BRANCH="$DEFAULT_GIT_BRANCH"
echo "PACKAGE_GIT_BRANCH set to value of DEFAULT_GIT_BRANCH"
fi
var="${PACKAGE_PREFIX}_REVISION"
if [[ -n "$PARAM_PACKAGE_REVISION" ]]; then
PACKAGE_REVISION="$PARAM_PACKAGE_REVISION"
echo "PACKAGE_REVISION passed from '-r'"
elif [[ -n "${!var}" ]]; then
PACKAGE_REVISION="${!var}"
echo "PACKAGE_REVISION set to value of ${var}"
fi
if [[ -z "$PACKAGE_REVISION" ]]; then
PACKAGE_REVISION="$DEFAULT_REVISION"
echo "PACKAGE_REVISION set to value of DEFAULT_REVISION"
fi
export PACKAGE_GIT_URL
export PACKAGE_GIT_BRANCH
export PACKAGE_REVISION
echo_bold "------------------------------------------------------------"
echo_bold "PACKAGE_GIT_URL: $PACKAGE_GIT_URL"
echo_bold "PACKAGE_GIT_BRANCH: $PACKAGE_GIT_BRANCH"
echo_bold "PACKAGE_REVISION: $PACKAGE_REVISION"
echo_bold "------------------------------------------------------------"
}
function create_workdir() {
check_env WORKDIR
logmust sudo rm -rf "$WORKDIR"
logmust mkdir "$WORKDIR"
logmust rm -f "$TOP/workdir"
logmust ln -s "$WORKDIR" "$TOP/workdir"
}
#
# apt install packages.
#
function install_pkgs() {
for attempt in {1..3}; do
echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
"apt-get install -y --allow-downgrades $*"
sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
-y --allow-downgrades "$@" && return
echo "apt-get install failed, retrying."
sleep 10
done
die "apt-get install failed after $attempt attempts"
}
#
# Install build dependencies listed in the debian/control file of the package.
#
function install_build_deps_from_control_file() {
logmust pushd "$WORKDIR/repo"
logmust sudo env DEBIAN_FRONTEND=noninteractive mk-build-deps --install \
--tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' \
debian/control
logmust popd
}
#
# Returns a list of all known packages in _RET_LIST.
#
function list_all_packages() {
local pkg
_RET_LIST=()
for pkg in "$TOP/packages/"*; do
pkg=$(basename "$pkg")
if [[ -f "$TOP/packages/$pkg/config.sh" ]]; then
_RET_LIST+=("$pkg")
fi
done
}
#
# Read a new-line separated list from a file, removing extra whitespace
# and comments. Return items in _RET_LIST.
#
function read_list() {
local file="$1"
local item
local line
_RET_LIST=()
[[ -f "$file" ]] || die "Not a file: $file"
while read -r line; do
# trim whitespace
item=$(echo "$line" | tr -d '[:space:]')
[[ -z "$item" ]] && continue
# ignore comments
[[ ${item:0:1} == "#" ]] && continue
_RET_LIST+=("$item")
done <"$file" || die "Failed to read list: $file"
}
#
# Read a package-list file and return listed packages in _RET_LIST.
#
function read_package_list() {
local file="$1"
local pkg
logmust read_list "$file"
for pkg in "${_RET_LIST[@]}"; do
check_package_exists "$pkg"
done
}
#
# List all target kernel packages. By default, it returns all the kernel
# flavors supported and built by linux-pkg, however this can be overridden
# via TARGET_KERNEL_FLAVORS, which can be useful when testing changes against
# a specific, or even mainline kernel.
#
function list_linux_kernel_packages() {
local kernel
_RET_LIST=()
if [[ -n "$TARGET_KERNEL_FLAVORS" ]]; then
for kernel in $TARGET_KERNEL_FLAVORS; do
(check_package_exists "linux-kernel-$kernel") ||
die "Invalid entry '$kernel' in TARGET_KERNEL_FLAVORS"
_RET_LIST+=("linux-kernel-$kernel")
done
else
for kernel in $SUPPORTED_KERNEL_FLAVORS; do
check_package_exists "linux-kernel-$kernel"
_RET_LIST+=("linux-kernel-$kernel")
done
fi
return 0
}
function install_shfmt() {
if [[ ! -f /usr/local/bin/shfmt ]]; then
local arch
arch=$(dpkg-architecture -q DEB_HOST_ARCH)
# The release names for shfmt don't use the actual
# architecture strings, unfortunately.
if [[ "$arch" == "arm64" ]]; then
arch="arm"
fi
logmust sudo wget -nv -O /usr/local/bin/shfmt \
https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_$arch
logmust sudo chmod +x /usr/local/bin/shfmt
fi
echo "shfmt version $(shfmt -version) is installed."
}
#
# Install kernel headers packages for all target kernels.
# The kernel packages are fetched from S3.
#
function install_kernel_headers() {
logmust determine_target_kernels
check_env KERNEL_VERSIONS DEPDIR
logmust list_linux_kernel_packages
# Note: linux packages returned in _RET_LIST
#
# On some platforms there are 2 headers packages and both must be
# installed. Here's an example on AWS:
# - linux-headers-5.3.0-1030-aws_5.3.0-1030.32~18.04.1_amd64.deb
# - linux-aws-5.3-headers-5.3.0-1030_5.3.0-1030.32~18.04.1_all.deb
#
local pkg
for pkg in "${_RET_LIST[@]}"; do
logmust install_pkgs "$DEPDIR/$pkg/"*-headers-*.deb
done
#
# Verify that headers are installed for all kernel versions
# stored in KERNEL_VERSIONS
#
local kernel
for kernel in $KERNEL_VERSIONS; do
logmust dpkg-query -l "linux-headers-$kernel" >/dev/null
done
}
function delphix_revision() {
#
# We use "delphix" in the default revision to make it easy to find all
# packages built by delphix installed on an appliance. This will be used
# along with DEFAULT_REVISION to get the full revision.
#
# We choose a timestamp as the second part since we want each package
# built to have a unique value for its full version, as new packages
# with the same full version as already installed ones would be skipped
# during an upgrade.
#
# Note that having revision numbers increasing monotonically is a
# requirement during regular upgrades. This is not a hard requirement for
# Delphix Appliance upgrades, however we prefer keeping things in-line
# with the established conventions.
#
echo "delphix.$(date '+%Y.%m.%d.%H.%M')"
}
function determine_dependencies_base_url() {
[[ -n "$DEPENDENCIES_BASE_URL" ]] && return
check_env DEFAULT_GIT_BRANCH
#
# The location of package artifacts depends on whether a release tag
# or a development branch is being built.
#
if is_release_branch; then
local version="${DEFAULT_GIT_BRANCH#release/}"
local url="s3://release-de-images/internal-artifacts/$version"
local suv
suv="$(aws s3 ls "$url/" | awk '{print $2}' | tr -d '/' | sort -V | tail -n 1)"
[[ -n "$suv" ]] || die "No artifacts found at $url"
DEPENDENCIES_BASE_URL="$url/$suv/input-artifacts/combined-packages/packages"
else
DEPENDENCIES_BASE_URL="s3://snapshot-de-images/builds/$JENKINS_OPS_DIR/linux-pkg/$DEFAULT_GIT_BRANCH/build-package"
fi
#
# Make sure the dependencies directory actually exists
#
logmust aws s3 ls "$DEPENDENCIES_BASE_URL"
export DEPENDENCIES_BASE_URL
}
function get_package_dependency_s3_url() {
local dep="$1"
logmust determine_dependencies_base_url
#
# The base url can be either pointing to:
# 1. Input artifacts from a previously built Delphix Appliance, or
# 2. A build-package/<package> directory that stores many builds of the
# same package and a "latest" link that points to the latest build
# for that package.
# In case 1 the artifacts are right there under the base url, whereas
# in case 2 we first need to dereference the "latest" link.
#
# The user can provide their own DEPENDENCIES_BASE_URL based on a
# previous Delphix Appliance build or have it determined automatically
# by determine_dependencies_base_url() based on the product branch
# being built.
#
if [[ "$DEPENDENCIES_BASE_URL" == */build-package ]]; then
local s3url="$DEPENDENCIES_BASE_URL/$dep/post-push"
(logmust aws s3 cp --only-show-errors "$s3url/latest" .) ||
die "Artifacts for dependency '$dep' missing." \
"Dependency must be built first."
logmust cat latest
local bucket="${DEPENDENCIES_BASE_URL#s3://}"
bucket=${bucket%%/*}
s3url="s3://$bucket/$(cat latest)"
logmust rm latest
_RET="$s3url"
else
_RET="$DEPENDENCIES_BASE_URL/$dep"
fi
}
#
# Fetch artifacts from S3 for all packages listed in PACKAGE_DEPENDENCIES which
# is defined in the package's config.
#
function fetch_dependencies() {
local source="$1"
export DEPDIR="$WORKDIR/dependencies"
logmust mkdir "$DEPDIR"
logmust cd "$DEPDIR"
if [[ -z "$PACKAGE_DEPENDENCIES" ]]; then
echo "Package has no linux-pkg dependencies to fetch."
return
fi
local dep s3urlvar s3url
for dep in $PACKAGE_DEPENDENCIES; do
echo "Fetching artifacts for dependency '$dep' ..."
get_package_prefix "$dep"
case "$source" in
"local")
logmust cp -r "$WORKDIR/../../$dep/tmp/artifacts/ $dep/"
;;
"s3")
s3urlvar="${_RET}_S3_URL"
if [[ -n "${!s3urlvar}" ]]; then
s3url="${!s3urlvar}"
echo "S3 URL of package dependency '$dep' provided externally"
echo "$s3urlvar=$s3url"
else
logmust get_package_dependency_s3_url "$dep"
s3url="$_RET"
fi
[[ "$s3url" != */ ]] && s3url="$s3url/"
logmust mkdir "$dep"
logmust aws s3 ls "$s3url"
logmust aws s3 cp --only-show-errors --recursive "$s3url" "$dep/"
echo_bold "Fetched artifacts for '$dep' from $s3url"
PACKAGE_DEPENDENCIES_METADATA="${PACKAGE_DEPENDENCIES_METADATA}$dep: $s3url\\n"
;;
*)
die "invalid source parameter specified: '$source'"
;;
esac
done
}
#
# Run git fetch with the passed arguments. Git url must be passed as first
# argument. If FETCH_GIT_TOKEN is set and this is a github repository
# then pass-in the token when fetching.
#
function git_fetch_helper() {
local orig_url="$1"
local git_url="$1"
local label=''
shift
if [[ -n "$FETCH_GIT_TOKEN" ]] &&
[[ "$git_url" == https://github.com/* ]]; then
git_url="${git_url/https:\/\//https:\/\/${FETCH_GIT_TOKEN}@}"
label='[token passed]'
fi
echo "Running: $label git fetch $orig_url $*"
git fetch "$git_url" "$@" || die "git fetch failed"
}
#
# Run git fetch with the passed arguments. Git url must be passed as first
# argument. If FETCH_GIT_TOKEN is set and this is a github repository
# then pass-in the token when fetching.
#
function git_branch_exists() {
local git_url="$1"
local label=''
shift
if [[ -n "$FETCH_GIT_TOKEN" ]] &&
[[ "$git_url" == https://github.com/* ]]; then
git_url="${git_url/https:\/\//https:\/\/${FETCH_GIT_TOKEN}@}"
label='[token passed]'
fi
git fetch "$git_url" "$@" --no-tags --depth=1 || return 1
}
#
# Fetch package repository into $WORKDIR/repo
#
function fetch_repo_from_git() {
check_env PACKAGE_GIT_URL PACKAGE_GIT_BRANCH DEFAULT_GIT_BRANCH
logmust mkdir "$WORKDIR/repo"
logmust cd "$WORKDIR/repo"
logmust git init
#
# If we are updating the package, we need to fetch both the
# main branch and the upstream branch with their histories.
# Otherwise just get the latest commit of the main branch.
#
if [[ "$DO_UPDATE_PACKAGE" == "true" ]]; then
logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
"+$PACKAGE_GIT_BRANCH:repo-HEAD"
logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
"+upstreams/$DEFAULT_GIT_BRANCH:upstream-HEAD"
logmust git show-ref repo-HEAD
logmust git show-ref upstream-HEAD
else
#
# The "os-upgrade" branch is used for Ubuntu version upgrade projects. As such, the following
# block of code will only be executed if the `PACKAGE_GIT_BRANCH` is set to "os-upgrade". Not
# all product repositories are required to have an "os-upgrade" branch, because some packages
# will successfully build on the new Ubuntu version as well as the old without any changes.
# To allow engineering to create the "os-upgrade" branch only in repositories that require
# changes for the new Ubuntu version, this block of code will check if the "os-upgrade" branch
# exists in the repository and if it does not, it will build off of "develop" instead.
#
if [[ "$PACKAGE_GIT_BRANCH" == "os-upgrade" ]]; then
git_branch_exists "$PACKAGE_GIT_URL" "$PACKAGE_GIT_BRANCH"
branch_exists=$?
if [[ $branch_exists -ne 0 ]]; then
echo "NOTE: The branch 'os-upgrade' does not exist for this repo. Using 'develop' instead."
PACKAGE_GIT_BRANCH="develop"
fi
fi
logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
"+$PACKAGE_GIT_BRANCH:repo-HEAD" --depth=1
logmust git show-ref repo-HEAD
fi
logmust git checkout repo-HEAD
}
function generate_commit_message_from_dsc() {
local dsc
shopt -s failglob
dsc=$(echo "$WORKDIR/source/$UPSTREAM_SOURCE_PACKAGE"*.dsc)
shopt -u failglob
rm -f "$WORKDIR/commit-message"
grep -E '^Version:' "$dsc" >"$WORKDIR/commit-message"
echo "" >>"$WORKDIR/commit-message"
cat "$dsc" >>"$WORKDIR/commit-message"
}
#
# Fetches branch upstreams/<branch> from our default repository into local
# upstream-HEAD branch, then attempts to update it with changes from
# the source package UPSTREAM_SOURCE_PACKAGE, fetched from apt.
#
function update_upstream_from_source_package() {
check_env PACKAGE_GIT_BRANCH UPSTREAM_SOURCE_PACKAGE
#
# Fetch the source package into source/
#
logmust mkdir "$WORKDIR/source"
logmust cd "$WORKDIR/source"
logmust apt-get source "$UPSTREAM_SOURCE_PACKAGE"
#
# Checkout the upstream branch from our repository, and delete all
# files.
#
logmust cd "$WORKDIR/repo"
logmust git checkout -q upstream-HEAD
logmust git rm -qrf .
logmust git clean -qfxd
#
# Deploy the files from the source package on top of our repo.
#
logmust cd "$WORKDIR"
shopt -s dotglob failglob
logmust mv source/"$UPSTREAM_SOURCE_PACKAGE"*/* repo/
shopt -u dotglob failglob
#
# Check if there are any changes. If so then commit them, and put the
# source package description as the commit message.
#
logmust cd "$WORKDIR/repo"
logmust git add -f .
if git diff --cached --quiet; then
echo "NOTE: upstream for $PACKAGE is already up-to-date."
else
logmust generate_commit_message_from_dsc
logmust git commit -F "$WORKDIR/commit-message"
logmust git show-ref upstream-HEAD
logmust touch "$WORKDIR/upstream-updated"
fi
logmust cd "$WORKDIR"
}
#
# Fetches branch upstreams/<branch> from our default repository into local
# upstream-HEAD branch, then attempts to update it with changes from
# the remote repository specified by UPSTREAM_GIT_URL and UPSTREAM_GIT_BRANCH.
#
function update_upstream_from_git() {
check_env UPSTREAM_GIT_URL UPSTREAM_GIT_BRANCH
logmust cd "$WORKDIR/repo"
#
# checkout our local branch that tracks upstream.
#
logmust git checkout -q upstream-HEAD
#
# Fetch updates from third-party upstream repository.
#
logmust git_fetch_helper "$UPSTREAM_GIT_URL" "$UPSTREAM_GIT_BRANCH"
#
# Compare third-party upstream repository to our local snapshot of the
# upstream repository.
#
if git diff --quiet FETCH_HEAD upstream-HEAD; then
echo "NOTE: upstream for $PACKAGE is already up-to-date."
else
#
# Note we do --ff-only here which will fail if upstream has
# been rebased. We always want this behaviour as a rebase
# is not something that maintainers usually do and if they do
# then we definitely want to be notified.
#
logmust git merge --no-edit --ff-only --no-stat FETCH_HEAD
logmust git show-ref upstream-HEAD
logmust touch "$WORKDIR/upstream-updated"
fi
logmust cd "$WORKDIR"
}
#
# Returns true if upstreams/<branch> needs to be merged into <branch> for the
# active package, where <branch> is the branch being updated, i.e.
# DEFAULT_GIT_BRANCH.
#
function is_merge_needed() {
local repo_ref="refs/heads/repo-HEAD"
local upstream_ref="refs/heads/upstream-HEAD"
logmust pushd "$WORKDIR/repo"
check_git_ref "$upstream_ref" "$repo_ref"
if git merge-base --is-ancestor "$upstream_ref" "$repo_ref"; then
echo "Upstream is already merged into repo-HEAD"
_RET=false
else
_RET=true
fi
logmust popd
}
#
# Default function for merging upstreams/<branch> into <branch>, where <branch>
# is the branch being updated, i.e. DEFAULT_GIT_BRANCH. Note that this function
# does not actually look at the upstream repository itself, but relies on
# local branches repo-HEAD and upstream-HEAD to be present.
#
# If merge was needed, file $WORKDIR/repo-updated is created and previous tip
# of <branch> is saved in repo-HEAD-saved. The repo-updated file lets the
# caller (typically Jenkins) know if a merge was necessary. The repo-HEAD-saved
# ref should be compared to the remote branch when it is time to push the
# merge; if they differ it means that the remote branch was modified and
# so the merge should be aborted -- this can happen if a PR was merged by a
# developer while auto-update was running.
#
function merge_with_upstream_default() {
local repo_ref="refs/heads/repo-HEAD"
local upstream_ref="refs/heads/upstream-HEAD"
logmust cd "$WORKDIR/repo"
check_git_ref "$upstream_ref" "$repo_ref"
logmust git checkout -q repo-HEAD
if git merge-base --is-ancestor "$upstream_ref" HEAD; then
echo "NOTE: $PACKAGE is already up-to-date with upstream."
return 0
fi
#
# Do a backup of the repo-HEAD branch so that it can be compared to the
# remote when time comes to do a push.
#
logmust git branch repo-HEAD-saved
logmust git merge --no-edit --no-stat "$upstream_ref"
logmust git show-ref repo-HEAD
logmust touch "$WORKDIR/repo-updated"
}
#
# Check if git credentials are set for pushing update. If running in
# interactive mode, it will prompt the user for credentials if they are not
# provided in env.
#
function check_git_credentials_set() {
if [[ -z "$PUSH_GIT_TOKEN" ]] && [[ -z "$PUSH_GIT_USER" || -z "$PUSH_GIT_PASSWORD" ]]; then
if [[ -t 1 ]]; then
if [[ "$DRYRUN" == "false" ]]; then
echo_bold "WARNING: this is NOT a dry-run, you are pushing to" \
"a production branch"
fi
echo "Please enter git token to push to remote ($DEFAULT_PACKAGE_GIT_URL)."
read -r -s -p "Token: " PUSH_GIT_TOKEN
export PUSH_GIT_TOKEN
else
die "PUSH_GIT_TOKEN or PUSH_GIT_USER & PUSH_GIT_PASSWORD must be set."
fi
fi
}