forked from scala/scala
-
Notifications
You must be signed in to change notification settings - Fork 0
/
bootstrap
596 lines (487 loc) · 26.4 KB
/
bootstrap
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
#!/bin/bash -e
# Script Overview
# - determine scala version
# - determine module versions
# - build minimal core (aka locker) of Scala, use the determined version number, publish to private-repo
# - build those modules where a binary compatible version doesn't exist, publish to private-repo
# - build Scala using the previously built core and bootstrap modules, publish to private-repo (overwrites the minimal core version on private-repo)
# - for releases (not nightlies)
# - stage Scala on sonatype
# - rebuild modules that needed a rebuild with this Scala build, and stage them on sonatype
# - for nightlies
# - force rebuild all modules and publish them locally (for testing purposes)
# - the Scala version is serialized to jenkins.properties, which is passed downstream to scala-release jobs
# - this removes the need to tag scala/scala-dist (it's still encouraged for releases, but not a hard requirement)
# Specifying the Scala version:
# - To build a release (this enables publishing to sonatype):
# - Either specify SCALA_VER_BASE. You may also specify SCALA_VER_SUFFIX, the Scala version is SCALA_VER=$SCALA_VER_BASE$SCALA_VER_SUFFIX.
# - Or have the current HEAD tagged as v$base$suffix
# - To prevent staging on sonatype (for testing), set publishToSonatype to anything but "yes"
# - Note: After building a release, the jenkins job provides an updated versions.properties file as artifact.
# Put this file in the Scala repo and create a pull request, and also update the file build.number.
#
# - Otherwise, a nightly release is built:
# - version number is read from the build.number file, extended with -$sha-nightly
# Specifying module versions: there are two modes
# - If moduleVersioning="versions.properties" (default): in this mode we use release versions for the modules.
# - Module versions are read from the versions.properties file.
# - Set <MODULE>_VER to override the default, e.g. XML_VER="1.0.4".
# - The git revision is set to <MODULE>_REF="v$<MODULE>_VER". Make sure the tag exists (you can't override <MODULE>_REF).
#
# - Otherwise (moduleVersioning has some other value): in this mode we use nightly version numbers for modules.
# - By default the script sets all <MODULE>_REF to "HEAD", override to build a specific revision.
# - The <MODULE>_VER is set to a nightly version, for example "1.0.3-7-g14888a2-nightly" (you can't override <MODULE>_VER)
# Modules are automatically built if necessary.
# - A module is built if it doesn't exist in the maven repository. Note that the lookup uses two versions:
# - The version of the module (see below how it's determined)
# - The binary version of of the SCALA_VER release that is being built
# - sbt computes the binary version when looking up / building modules (*). Examples:
# - 2.12.0-M1, 2.12.0-RC3: the full version is used
# - 2.12.0, 2.12.1-M1, 2.12.1-RC3, 2.12.1: the binary version 2.12 is used
#
# - Example: assume that `scala-xml_2.11 % 1.0.3` and `scala-xml_2.12.0-M1 % 1.0.3` both exists
# - XML_VER=1.0.3 and SCALA_VER=2.11.7 => no rebuild (binary version remains 2.11)
# - XML_VER=1.0.3 and SCALA_VER=2.12.0-M2 => rebuild (new binary version 2.12.0-M2)
# - XML_VER=1.0.4 and SCALA_VER=2.11.7 => rebuild (new version for the module, not yet on maven)
# NOTE: this is not the recommended way of publishing a module. Instead, prefer to release `scala-xml_2.11 % 1.0.4`
# using the existing scala 2.11.6 compiler before releasing 2.11.7. Sometimes it's necessary though. One
# example was 2.11.1, which contained a fix in the backend (SerialVersionUID was ignored). All modules needed
# to be re-built using the 2.11.1 release, we could not use 2.11.0. We could also not release the modules
# after 2.11.1 was out, because that way the scala-library-all pom of 2.11.1 would depend on the old modules.
#
# (*) https://github.com/sbt/sbt/blob/0.13.8/util/cross/src/main/input_sources/CrossVersionUtil.scala#L39
# Binary incompatible changes in Modules: example with Scala 2.11 / 2.12 and scala-parser-combinators
# - The 1.0.x branch on scala-parser-combinators remains binary compatible with 1.0.0
# - Scala 2.11 will always use 1.0.x releases: we ship scala-parser-combinators with the distribution,
# so we cannot introduce incompatible changes in a minor release.
# - The master branch of scala-parser-combinators contains binary incompatible changes, versioned 1.1.x
# - Scala 2.12 will use 1.1.x releases
# - No changes to the build script required: just put the 1.1.x version number into versions.properties
#
# Note: It's still OK for a module to release a binary incompatible version to maven, for example
# scala-parser-combinators_2.11 % 1.1.0. Users can depend on this in their sbt build. But for the
# distribution (tar/zip archives, scala-library-all) we have to stay on the binary compatible version.
# Requirements
# - SBT_CMD must point to sbt from sbt-extras
# - ~/.sonatype-curl, ~/.m2/settings.xml, ~/.credentials, ~/.credentials-sonatype, ~/.credentials-private-repo
# as defined by https://github.com/scala/scala-jenkins-infra/tree/master/templates/default
# - ~/.sbt/0.13/plugins/gpg.sbt with:
# addSbtPlugin("com.typesafe.sbt" % "sbt-pgp" % "0.8.1")
# Note: private-repo used to be private-repo.typesafe.com. now we're running artifactory on scala-ci.typesafe.com/artifactory
moduleVersioning=${moduleVersioning-"versions.properties"}
publishPrivateTask=${publishPrivateTask-"publish"}
publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"}
publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"}
forceRebuild=${forceRebuild-no}
sbtBuildTask="testAll" # TESTING leave empty to avoid the sanity check
clean="clean" # TESTING leave empty to speed up testing
baseDir=${WORKSPACE-`pwd`}
scriptsDir="$baseDir/scripts"
. $scriptsDir/common
# we must change ivy home to get a fresh ivy cache, otherwise we get half-bootstrapped scala
# rm it in case it existed (and there's no ivy2-shadow, which indicates we're running in a TESTING environment)...
# we don't nuke the whole ws since that clobbers the git clones needlessly
[[ -d $baseDir/ivy2-shadow ]] || rm -rf $baseDir/ivy2
mkdir -p $baseDir/ivy2
rm -rf $baseDir/resolutionScratch_
mkdir -p $baseDir/resolutionScratch_
# repo used to publish "locker" scala to (to start the bootstrap)
releaseTempRepoCred="private-repo"
releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"}
generateRepositoriesConfig $releaseTempRepoUrl
##### git
gfxd() {
git clean -fxd # TESTING
}
update() {
[[ -d $baseDir ]] || mkdir -p $baseDir
cd $baseDir
if [ ! -d $baseDir/$2 ]; then git clone "https://github.com/$1/$2.git"; fi
cd $2
git fetch --tags "https://github.com/$1/$2.git"
(git fetch "https://github.com/$1/$2.git" $3 && git checkout -fq FETCH_HEAD) #|| git checkout -fq $3 # || fallback is for local testing on tag
git reset --hard
}
##### sonatype interface
stApi="https://oss.sonatype.org/service/local"
function st_curl(){
curl -H "Content-Type: application/json" -H "accept: application/json,application/vnd.siesta-error-v1+json,application/vnd.siesta-validation-errors-v1+json" -K ~/.sonatype-curl -s -o - $@
}
function st_stagingReposOpen() {
st_curl "$stApi/staging/profile_repositories" | jq '.data[] | select(.profileName == "org.scala-lang") | select(.type == "open")'
}
function st_stagingRepoDrop() {
repo=$1
message=$2
echo "{\"data\":{\"description\":\"$message\",\"stagedRepositoryIds\":[\"$repo\"]}}" | st_curl -X POST -d @- "$stApi/staging/bulk/drop"
}
function st_stagingRepoClose() {
repo=$1
message=$2
echo "{\"data\":{\"description\":\"$message\",\"stagedRepositoryIds\":[\"$repo\"]}}" | st_curl -X POST -d @- "$stApi/staging/bulk/close"
}
# ARGH trying to get this to work on multiple versions of sbt-extras...
# the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir
# the new version of sbt-extras ignores sbt-dir, so we pass it in as -Dsbt.global.base
# need to set sbt-dir to one that has the gpg.sbt plugin config
sbtArgs="-ivy $baseDir/ivy2 -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig -Dsbt.global.base=$HOME/.sbt/0.13 -sbt-dir $HOME/.sbt/0.13"
sbtBuild() {
echo "### sbtBuild: "$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@"
$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1
}
sbtResolve() {
cd $baseDir/resolutionScratch_
touch build.sbt
# Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin used to.
cross=${4-binary}
echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" \
"set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \
'show update' >> $baseDir/logs/resolution 2>&1
}
# Oh boy... can't use scaladoc to document scala-xml/scala-parser-combinators
# if scaladoc depends on the same version of scala-xml/scala-parser-combinators.
# Even if that version is available through the project's resolvers, sbt won't look past this project.
# SOOOOO, we set the version to a dummy (-DOC), generate documentation,
# then set the version to the right one and publish (which won't re-gen the docs).
# Also tried publish-local without docs using 'set publishArtifact in (Compile, packageDoc) := false' and republishing, no dice.
# Each buildModule() function is invoked twice: first to build against locker and publish to private-repo, then
# to build against the release and publish to sonatype (or publish-local if publishToSonatype is not "yes").
# In the second round, sbtResolve is always true: the module will be found in the private-repo!
# Therefore, if MODULE_BUILT is "yes" (in the second round), we know that we need to build (and publish) the
# module again.
#
# Note: we tried an alternative solution in which sbtResolve would not look at private-repo, but that fails. For example,
# scala-xml depends on scala-library, so sbt tries to find the scala-library of the version that we are currently building,
# which exists only in private-repo.
docTask() {
if [[ "$STARR_REF" != "" && "$1" != "yes" ]]; then
# Don't build module docs on the first round of module builds when bootstrapping
# a binary incompatible compiler change to avoid linkage errors with using the old Scaladoc
echo set publishArtifact in packageDoc in Compile := false
else
echo doc
fi
}
buildXML() {
if [ "$XML_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-xml" $XML_VER )
then echo "Found scala-xml $XML_VER; not building."
else
update scala scala-xml "$XML_REF" && gfxd
doc="$(docTask $XML_BUILT)"
sbtBuild 'set version := "'$XML_VER'-DOC"' $clean "$doc" 'set version := "'$XML_VER'"' test "${buildTasks[@]}"
XML_BUILT="yes" # ensure the module is built and published when buildXML is invoked for the second time, see comment above
fi
}
buildParsers() {
if [ "$PARSERS_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-parser-combinators" $PARSERS_VER )
then echo "Found scala-parser-combinators $PARSERS_VER; not building."
else
update scala scala-parser-combinators "$PARSERS_REF" && gfxd
doc="$(docTask $PARSERS_BUILT)"
sbtBuild 'set version := "'$PARSERS_VER'-DOC"' $clean "$doc" 'set version := "'$PARSERS_VER'"' test "${buildTasks[@]}"
PARSERS_BUILT="yes"
fi
}
buildPartest() {
if [ "$PARTEST_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-partest" $PARTEST_VER )
then echo "Found scala-partest $PARTEST_VER; not building."
else
update scala scala-partest "$PARTEST_REF" && gfxd
doc="$(docTask $PARTEST_BUILT)"
sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' $clean "$doc" test "${buildTasks[@]}"
PARTEST_BUILT="yes"
fi
}
buildSwing() {
if [ "$SWING_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-swing" $SWING_VER )
then echo "Found scala-swing $SWING_VER; not building."
else
update scala scala-swing "$SWING_REF" && gfxd
doc="$(docTask $SWING_BUILT)"
sbtBuild 'set version := "'$SWING_VER'"' $clean "$doc" test "${buildTasks[@]}"
SWING_BUILT="yes"
fi
}
# should only be called with publishTasks publishing to private-repo
buildScalacheck(){
if [ "$SCALACHECK_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scalacheck" "scalacheck" $SCALACHECK_VER )
then echo "Found scalacheck $SCALACHECK_VER; not building."
else
update rickynils scalacheck $SCALACHECK_REF && gfxd
doc="$(docTask $SCALACHECK_BUILT)"
sbtBuild 'set version := "'$SCALACHECK_VER'"' 'set VersionKeys.scalaParserCombinatorsVersion := "'$PARSERS_VER'"' $clean "$doc" publish # test times out NOTE: never published to sonatype
SCALACHECK_BUILT="yes"
fi
}
# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to private-repo)
buildModules() {
publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"private-repo\" at \"$releaseTempRepoUrl\")")
buildTasks=($publishPrivateTask)
buildXML
buildParsers
buildSwing
# buildScalacheck
buildPartest
}
buildPublishedModules() {
publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)")
buildTasks=($publishSonatypeTaskModules)
buildXML
buildParsers
buildSwing
buildPartest
}
## BUILD STEPS:
scalaVerToBinary() {
# $1 = SCALA_VER
# $2 = SCALA_VER_BASE
# $3 = SCALA_VER_SUFFIX
local RE='\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)'
local majMin="$(echo $2 | sed -e "s#$RE#\1.\2#")"
local patch="$(echo $2 | sed -e "s#$RE#\3#")"
# The binary version is majMin (e.g. "2.12") if
# - there's no suffix : 2.12.0, 2.12.1
# - the suffix starts with "-bin": 2.12.0-bin-M1
# - the patch version is > 0 : 2.12.1-M1, 1.12.3-RC2, 2.12.1-sha-nightly, 2.12.2-SNAPSHOT
#
# Otherwise, the binary version is the full version: 2.12.0-M1, 2.12.0-RC2, 2.12.0-sha-nightly, 2.12.0-SNAPSHOT
#
# Adapted from sbt: https://github.com/sbt/sbt/blob/0.13.8/util/cross/src/main/input_sources/CrossVersionUtil.scala#L39
#
# Note: during the pre-release cycle of a major release (e.g. before 2.12.0), the SCALA_BINARY_VER of nightly / SNAPSHOT
# versions is the full version, e.g. 2.12.0-sha-nightly, so modules are always re-built. This is in line with what sbt
# does: for example, with scalaVersion := "2.12.0-SNAPSHOT", sbt will resolve scala-xml as scala-xml_2.12.0-SNAPSHOT.
# Once the 2.12.0 release is out, the binary version is 2.12 for all versions (e.g. for 2.12.1-sha-nightly).
if [[ "$3" == "" || "${3:0:4}" == "-bin" || "$patch" != "0" ]]; then
echo "$majMin"
else
echo "$1"
fi
}
determineScalaVersion() {
cd $WORKSPACE
parseScalaProperties "versions.properties"
echo "repo_ref=2.12.x" >> $baseDir/jenkins.properties # for the -dist downstream jobs that build the actual archives
# each of the branches below defines the following vars: SCALA_VER_BASE, SCALA_VER_SUFFIX, SCALADOC_SOURCE_LINKS_VER, publishToSonatype
if [ -z "$SCALA_VER_BASE" ]; then
echo "No SCALA_VER_BASE specified."
scalaTag=$(git describe --tag --exact-match ||:)
if [ -z "$scalaTag" ]
then
echo "No tag found, building nightly snapshot."
$SBT_CMD $sbtArgs 'set baseVersionSuffix in Global := "SHA-NIGHTLY"' generateBuildCharacterPropertiesFile
parseScalaProperties "buildcharacter.properties"
SCALA_VER_BASE="$maven_version_base"
SCALA_VER_SUFFIX="$maven_version_suffix"
SCALADOC_SOURCE_LINKS_VER=$(git rev-parse HEAD)
# TODO: publish nightly snapshot using this script - currently it's a separate jenkins job still running at EPFL.
publishToSonatype="no"
else
echo "HEAD is tagged as $scalaTag."
# borrowed from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
local RE='v*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' # don't change this to make it more accurate, it's not worth it
SCALA_VER_BASE="$(echo $scalaTag | sed -e "s#$RE#\1.\2.\3#")"
SCALA_VER_SUFFIX="$(echo $scalaTag | sed -e "s#$RE#\4#")"
SCALADOC_SOURCE_LINKS_VER=$scalaTag
if [ "$SCALA_VER_BASE" == "$scalaTag" ]; then
echo "Could not parse version $scalaTag"
exit 1
fi
publishToSonatype=${publishToSonatype-"yes"} # unless forced previously, publish
fi
else
publishToSonatype=${publishToSonatype-"yes"} # unless forced previously, publish
# if version base/suffix are provided, we assume a corresponding tag exists for the scaladoc source links
SCALADOC_SOURCE_LINKS_VER="v$SCALA_VER_BASE$SCALA_VER_SUFFIX"
fi
SCALA_VER="$SCALA_VER_BASE$SCALA_VER_SUFFIX"
SCALA_BINARY_VER=$(scalaVerToBinary $SCALA_VER $SCALA_VER_BASE $SCALA_VER_SUFFIX)
echo "version=$SCALA_VER" >> $baseDir/jenkins.properties
echo "sbtDistVersionOverride=-Dproject.version=$SCALA_VER" >> $baseDir/jenkins.properties
scalaVersionTasks=('set every scalaVersion := "'$SCALA_VER'"')
echo "Building Scala $SCALA_VER."
}
deriveVersion() {
update $1 $2 $3 &> /dev/null
echo "$(git describe --tag --match=v* | cut -dv -f2)-nightly"
}
deriveVersionAnyTag() {
update $1 $2 $3 &> /dev/null
echo "$(git describe --tag | cut -dv -f2)-nightly"
}
# determineScalaVersion must have been called
deriveModuleVersions() {
if [ "$moduleVersioning" == "versions.properties" ]; then
# use versions.properties as defaults when no version specified on the command line
XML_VER=${XML_VER-$scala_xml_version_number}
PARSERS_VER=${PARSERS_VER-$scala_parser_combinators_version_number}
SWING_VER=${SWING_VER-$scala_swing_version_number}
PARTEST_VER=${PARTEST_VER-$partest_version_number}
SCALACHECK_VER=${SCALACHECK_VER-$scalacheck_version_number}
XML_REF="v$XML_VER"
PARSERS_REF="v$PARSERS_VER"
SWING_REF="v$SWING_VER"
PARTEST_REF="v$PARTEST_VER"
SCALACHECK_REF="$SCALACHECK_VER" # no `v` in their tags
else
# use HEAD as default when no revision is specified on the command line
XML_REF=${XML_REF-"HEAD"}
PARSERS_REF=${PARSERS_REF-"HEAD"}
SWING_REF=${SWING_REF-"HEAD"}
PARTEST_REF=${PARTEST_REF-"HEAD"}
SCALACHECK_REF=${SCALACHECK_REF-"HEAD"}
XML_VER=$(deriveVersion scala scala-xml "$XML_REF")
PARSERS_VER=$(deriveVersion scala scala-parser-combinators "$PARSERS_REF")
SWING_VER=$(deriveVersion scala scala-swing "$SWING_REF")
PARTEST_VER=$(deriveVersion scala scala-partest "$PARTEST_REF")
SCALACHECK_VER=$(deriveVersionAnyTag rickynils scalacheck "$SCALACHECK_REF")
fi
echo "Module versions (versioning strategy: $moduleVersioning):"
echo "PARSERS = $PARSERS_VER at $PARSERS_REF"
echo "PARTEST = $PARTEST_VER at $PARTEST_REF"
# echo "SCALACHECK = $SCALACHECK_VER at $SCALACHECK_REF"
echo "SWING = $SWING_VER at $SWING_REF"
echo "XML = $XML_VER at $XML_REF"
}
createNetrcFile() {
local netrcFile=$HOME/`basename $1`-netrc
grep 'host=' $1 | sed 's/host=\(.*\)/machine \1/' > $netrcFile
grep 'user=' $1 | sed 's/user=\(.*\)/login \1/' >> $netrcFile
grep 'password=' $1 | sed 's/password=\(.*\)/password \1/' >> $netrcFile
}
removeExistingBuilds() {
createNetrcFile "$HOME/.credentials-private-repo"
local netrcFile="$HOME/.credentials-private-repo-netrc"
local storageApiUrl=`echo $releaseTempRepoUrl | sed 's/\(scala-release-temp\)/api\/storage\/\1/'`
local scalaLangModules=`curl -s $storageApiUrl/org/scala-lang | jq -r '.children | .[] | "org/scala-lang" + .uri' | grep -v actors-migration`
for module in $scalaLangModules; do
local artifacts=`curl -s $storageApiUrl/$module | jq -r ".children | .[] | select(.uri | contains(\"$SCALA_VER\")) | .uri"`
for artifact in $artifacts; do
echo "Deleting $releaseTempRepoUrl$module$artifact"
curl -s --netrc-file $netrcFile -X DELETE $releaseTempRepoUrl$module$artifact
done
done
}
constructUpdatedModuleVersions() {
updatedModuleVersions=()
# force the new module versions for building the core. these may be different from the values in versions.properties,
# either because the variables (XML_VER) were provided, or because we're building the modules from HEAD.
# in the common case, the values are the same as in versions.properties.
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-parser-combinators.version.number=$PARSERS_VER")
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-swing.version.number=$SWING_VER")
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-xml.version.number=$XML_VER")
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dpartest.version.number=$PARTEST_VER")
# updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscalacheck.version.number=$SCALACHECK_VER")
# allow overriding the jline version using a jenkins build parameter
if [ ! -z "$JLINE_VER" ] ; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Djline.version=$JLINE_VER"); fi
if [ ! -z "$SCALA_BINARY_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.binary.version=$SCALA_BINARY_VER"); fi
}
# build locker (scala + modules) and quick, publishing everything to private-repo
bootstrap() {
echo "### Bootstrapping"
cd $WORKSPACE
#### (Optional) STARR.
if [ ! -z "$STARR_REF" ]; then
echo "### Building STARR"
STARR_DIR=./scala-starr
STARR_VER_SUFFIX="-$(git rev-parse --short $STARR_REF)-nightly"
STARR_VER=$SCALA_VER_BASE$STARR_VER_SUFFIX
rm -rf "$STARR_DIR"
(
git clone --reference $WORKSPACE/.git $WORKSPACE/.git $STARR_DIR
cd $STARR_DIR
git co $STARR_REF
$SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $releaseTempRepoUrl $STARR_VER" $clean publish >> $baseDir/logs/builds 2>&1
)
fi
#### LOCKER
echo "### Building locker"
# for bootstrapping, publish core (or at least smallest subset we can get away with)
# so that we can build modules with this version of Scala and publish them locally
# must publish under $SCALA_VER so that the modules will depend on this (binary) version of Scala
# publish more than just core: partest needs scalap
# in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler
if [ ! -z "$STARR_VER" ]; then SET_STARR=-Dstarr.version=$STARR_VER; fi
$SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $releaseTempRepoUrl $SCALA_VER" $clean publish >> $baseDir/logs/builds 2>&1
echo "### Building modules using locker"
# build, test and publish modules with this core
# publish to our internal repo (so we can resolve the modules in the scala build below)
# we only need to build the modules necessary to build Scala itself
# since the version of locker and quick are the same
buildModules
constructUpdatedModuleVersions
#### QUICK
echo "### Bootstrapping Scala using locker"
# # TODO: close all open staging repos so that we can be reasonably sure the only open one we see after publishing below is ours
# # the sbt call will create a new one
#
# Rebuild Scala with these modules so that all binary versions are consistent.
# Update versions.properties to new modules.
# Sanity check: make sure the Scala test suite passes / docs can be generated with these modules.
# don't skip locker (-Dlocker.skip=1), or stability will fail
# overwrite "locker" version of scala at private-repo with bootstrapped version
cd $baseDir
rm -rf build/
$SBT_CMD $sbtArgs \
--warn \
-Dstarr.version=$SCALA_VER \
${updatedModuleVersions[@]} \
"setupBootstrapQuick $releaseTempRepoUrl $SCALA_VER" \
$clean \
$sbtBuildTask \
dist/mkQuick \
publish
# clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala
rm -rf $baseDir/ivy2
# Run stability tests using the just built version as "quick" and a new version as "strap"
mv build/quick quick1
rm -rf build/
$SBT_CMD $sbtArgs \
--warn \
-Dstarr.version=$SCALA_VER \
${updatedModuleVersions[@]} \
"setupBootstrapQuick $releaseTempRepoUrl $SCALA_VER" \
$clean \
dist/mkQuick
mv build/quick build/strap
mv quick1 build/quick
$scriptsDir/stability-test.sh
# TODO: create PR with following commit (note that release will have been tagged already)
# git commit versions.properties -m"Bump versions.properties for $SCALA_VER."
}
# assumes we just bootstrapped, and current directory is $baseDir
# publishes locker to sonatype, then builds modules again (those for which version numbers were provided),
# and publishes those to sonatype as well
# finally, the staging repos are closed
publishSonatype() {
# stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary,
# since we're just publishing an existing build
echo "### Publishing core to sonatype"
$SBT_CMD $sbtArgs \
--warn \
-Dstarr.version=$SCALA_VER \
${updatedModuleVersions[@]} \
"setupBootstrapPublish $releaseTempRepoUrl $SCALA_VER" \
$publishSonatypeTaskCore
echo "### Publishing modules to sonatype"
# build/test/publish scala core modules to sonatype (this will start a new staging repo)
# (was hoping we could make everything go to the same staging repo, but it's not timing that causes two staging repos to be opened)
# NOTE: only publish those for which versions are set
# test and publish to sonatype, assuming you have ~/.sbt/0.13/sonatype.sbt and ~/.sbt/0.13/plugin/gpg.sbt
buildPublishedModules
open=$(st_stagingReposOpen)
allOpenUrls=$(echo $open | jq '.repositoryURI' | tr -d \")
allOpen=$(echo $open | jq '.repositoryId' | tr -d \")
echo "Closing open repos: $allOpen"
for repo in $allOpen; do st_stagingRepoClose $repo; done
echo "Closed sonatype staging repos: $allOpenUrls."
}
#### MAIN
determineScalaVersion
deriveModuleVersions
removeExistingBuilds
bootstrap
if [ "$publishToSonatype" == "yes" ]
then publishSonatype
fi