Skip to content

Commit

Permalink
Build base tarball (#2170)
Browse files Browse the repository at this point in the history
* Add Artifacts text file to packages output

* Add build of bare tarball plush prep.sh

* Update CI to build base tarball

* Build BaseTarball as part of centos71 Offline

* Build base tarball with centos71 online

* Add missing single quote

* Change permissions on prep.sh file

* Fixes based on PR comments
  • Loading branch information
dseefeld authored May 21, 2021
1 parent 8a18c55 commit 5dcc817
Show file tree
Hide file tree
Showing 9 changed files with 190 additions and 3 deletions.
2 changes: 1 addition & 1 deletion .vsts.pipelines/builds/matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ stages:
# Temporarily disable unit tests until entire product builds.
# See https://github.com/dotnet/source-build/issues/1549
Production: { generatePrebuiltBurndown: true, runUnitTests: false }
Online: { type: Online }
Online: { type: Online BaseTarball }
Offline: { type: Offline }
Offline Portable: { type: Offline Portable }

Expand Down
29 changes: 29 additions & 0 deletions .vsts.pipelines/jobs/ci-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ jobs:
stagingDirectory: $(rootDirectory)/sb/staging
systemLibunwind: ${{ parameters.systemLibunwind }}
tarballName: tarball_$(Build.BuildId)
baseTarballName: base_tarball_$(Build.BuildId)
tarballDownloadArgs: >-
/p:DownloadSourceBuildReferencePackagesTimeoutSeconds=600
/p:DownloadSourceBuiltArtifactsTimeoutSeconds=1500
Expand Down Expand Up @@ -121,6 +122,21 @@ jobs:
timeoutInMinutes: 45
continueOnError: true
# Create base tarball.
- script: |
set -ex
df -h
args="--skip-build --base-tarball"
if [ "$(reportPrebuiltLeaks)" = "true" ]; then
args="$args --enable-leak-detection"
fi
$(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(SetInternalPackageFeedPatDockerArg) $(imageName) ./build-source-tarball.sh \
"/tb/$(baseTarballName)" \
$args
du -h $(rootDirectory) | sort -h | tail -n 50
displayName: Create base tarball
condition: and(succeeded(), eq(variables['sb.baseTarball'], true))
# Create tarball.
- script: |
set -ex
Expand Down Expand Up @@ -206,6 +222,19 @@ jobs:
displayName: Copy tarball to output
condition: and(succeeded(), eq(variables['sb.tarball'], true))
# tar the base tarball directory into the drop directory.
- script: |
set -ex
df -h
$(docker.run) $(docker.tb.map) $(docker.drop.map) $(docker.tb.work) $(imageName) /bin/bash -c '
mkdir -p /drop/tarball/
# Make one .tar.gz for build, another for extras necessary to smoke test:
tar --numeric-owner -zcf "/drop/tarball/$(baseTarballName).tar.gz" "$(baseTarballName)"'
du -h $(rootDirectory) | sort -h | tail -n 50
displayName: Copy base tarball to output
condition: and(succeeded(), eq(variables['sb.baseTarball'], true))
# Build tarball.
- script: |
set -ex
Expand Down
5 changes: 5 additions & 0 deletions .vsts.pipelines/steps/calculate-config-flags.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,18 @@ steps:
setvariable sb.configuration Release
setvariable sb.portable false
setvariable sb.baseTarball false
setvariable sb.tarball false
setvariable sb.tarballOffline false
echo "Setting up configuration variables for CI type '$TYPE'"
for x in $TYPE; do
case $x in
BaseTarball)
setvariable sb.baseTarball true
;;
Debug|Release)
setvariable sb.configuration $x
;;
Expand Down
2 changes: 2 additions & 0 deletions Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@
<SourceBuiltArtifactsTarballUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/</SourceBuiltArtifactsTarballUrl>
<ReferencePackagesTarballName>Private.SourceBuild.ReferencePackages</ReferencePackagesTarballName>
<ReferencePackagesTarballUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core/assets/</ReferencePackagesTarballUrl>
<ArchiveArtifactsTextFileName>archiveArtifacts.txt</ArchiveArtifactsTextFileName>
<ArchiveArtifactsTextFile>$(ExternalTarballsDir)$(ArchiveArtifactsTextFileName)</ArchiveArtifactsTextFile>
<BaselineDataFile>$(ToolsLocalDir)prebuilt-baseline-</BaselineDataFile>
<OfflineBaselineDataFile>$(BaselineDataFile)offline.xml</OfflineBaselineDataFile>
<OnlineBaselineDataFile>$(BaselineDataFile)online.xml</OnlineBaselineDataFile>
Expand Down
11 changes: 11 additions & 0 deletions build-source-tarball.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ usage() {
echo " --with-ref-packages <dir> use the specified directory of available reference packages to determine what prebuilts to delete, instead of downloading the most recent version"
echo " --with-packages <dir> use the specified directory of available previously-built packages to determine what prebuilts to delete, instead of downloading the most recent version"
echo " --with-sdk <dir> use the specified SDK to check out source code. do not copy it to the tarball. an external SDK will be required to build from the tarball."
echo " --base-tarball creates a tarball without sdk, previously-build packages and reference packages"
echo "use -- to send the remaining arguments to build.sh"
}

Expand All @@ -23,6 +24,7 @@ TARBALL_ROOT=$1
shift

SKIP_BUILD=0
BASE_TARBALL=0
CUSTOM_SDK_DIR=''
MINIMIZE_DISK_USAGE=0
CUSTOM_REF_PACKAGES_DIR=''
Expand Down Expand Up @@ -79,6 +81,10 @@ while :; do
MAIN_BUILD_ARGS+=( "/p:CustomPreviouslySourceBuiltPackagesDir=$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" )
shift
;;
--base-tarball)
BASE_TARBALL=1
MAIN_BUILD_ARGS+=( "/p:PackBaseTarball=true" )
;;
--)
shift
echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
Expand All @@ -98,6 +104,11 @@ while :; do
shift
done

if [[ $BASE_TARBALL -eq 1 && ( "$CUSTOM_SDK_DIR" != "" || "$CUSTOM_REF_PACKAGES_DIR" != "" || "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" != "" ) ]]; then
echo "ERROR: Cannot specify --base-tarball with other --with* parameters (--with-sdk --with-ref-packages --with-packages)"
exit 1
fi

if [ $MINIMIZE_DISK_USAGE -eq 1 ]; then
echo "WARNING"
echo "WARNING"
Expand Down
15 changes: 15 additions & 0 deletions build.proj
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,21 @@
<RemoveDir Directories="$(BaseOutputPath)" />
</Target>

<Target Name="OutputArchiveArtifactsTextFile"
AfterTargets="Build"
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT'" >

<ItemGroup>
<ArtifactUrls Include="$(ReferencePackagesTarballUrl)$(ReferencePackagesTarballName).$(PrivateSourceBuildReferencePackagesPackageVersion).tar.gz" />
<ArtifactUrls Include="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz" />
</ItemGroup>
<WriteLinesToFile
File="$(ArchiveArtifactsTextFile)"
Lines="@(ArtifactUrls)"
Overwrite="true" />
</Target>


<Target Name="DownloadSourceBuildReferencePackages"
AfterTargets="Build"
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'"
Expand Down
15 changes: 13 additions & 2 deletions eng/SourceBuild.Tarball.targets
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@
$(DotNetCliToolDir)shared\2.1.0\**;
$(DotNetCliToolDir)tools\**;
"
Condition="'$(CustomDotNetSdkDir)' == ''"/>
Condition=" '$(CustomDotNetSdkDir)' == '' AND '$(PackBaseTarball)' != 'true' "/>

<TarballCopyFile
Include="@(TarballRootToolFile)"
Expand All @@ -155,6 +155,11 @@
<TarballCopyFile
Include="$(ProjectDir)support/tarball/build.sh"
RelativeDestination="build.sh" />

<TarballCopyFile
Condition=" '$(PackBaseTarball)' == 'true' "
Include="$(ProjectDir)support/tarball/prep.sh"
RelativeDestination="prep.sh" />

<!--
When building an internal build, the main smoke-test nuget config has authenticated sources
Expand Down Expand Up @@ -280,15 +285,21 @@
DependsOnTargets="AddTarballPackages">
<ItemGroup>
<ExternalTarballFile
Condition=" '$(PackBaseTarball)' != 'true' "
Include="$(ExternalTarballsDir)*.tar.gz"
RelativeRoot="packages\archive\" />

<!-- If not using previously source built artifacts tar.gz, use Production build output. -->
<ExternalTarballFile
Condition="'$(PrivateSourceBuiltArtifactsPackageVersion)' == ''"
Condition=" '$(PrivateSourceBuiltArtifactsPackageVersion)' == '' AND '$(PackBaseTarball)' != 'true' "
Include="@(PrivateSourceBuiltArtifactsTarGzFile)"
RelativeRoot="packages\archive\" />

<ExternalTarballFile
Condition=" '$(PackBaseTarball)' == 'true' "
Include="$(ArchiveArtifactsTextFile)"
RelativeRoot="packages\archive\" />

<TarballCopyFile Include="@(ExternalTarballFile);" />
</ItemGroup>
</Target>
Expand Down
25 changes: 25 additions & 0 deletions support/tarball/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,31 @@ while :; do
shift
done

if [ -f "$SCRIPT_ROOT/packages/archive/archiveArtifacts.txt" ]; then
ARCHIVE_ERROR=0
if [ ! -d "$SCRIPT_ROOT/.dotnet" ] && [ "$CUSTOM_SDK_DIR" == "" ]; then
echo "ERROR: SDK not found at $SCRIPT_ROOT/.dotnet"
ARCHIVE_ERROR=1
fi
if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" == "" ]; then
echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-packages parameter"
ARCHIVE_ERROR=1
fi
if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuild.ReferencePackages*.tar.gz ] && [ "$CUSTOM_REF_PACKAGES_DIR" == "" ]; then
echo "ERROR: Private.SourceBuild.ReferencePackages artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-ref-packages parameter"
ARCHIVE_ERROR=1
fi
if [ $ARCHIVE_ERROR == 1 ]; then
echo ""
echo " Errors detected in tarball. To prep the tarball, run prep.sh while online to install an SDK,"
echo " a Private.SourceBuilt.Artifacts tarball and a Private.SourceBuild.ReferencePackages tarball."
echo " After prepping the tarball, the tarball can be built offline. As an alternative to prepping"
echo " the tarball, these assets can be provided using the --with-sdk, --with-packages and"
echo " --with-ref-packages parameters."
exit 1
fi
fi

if [ -d "$CUSTOM_SDK_DIR" ]; then
export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version`
export CLI_ROOT="$CUSTOM_SDK_DIR"
Expand Down
89 changes: 89 additions & 0 deletions support/tarball/prep.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"

usage() {
echo "usage: $0"
echo ""
echo " Prepares a tarball to be built by downloading Private.SourceBuild.ReferencePackages.*.tar.gz and"
echo " Private.SourceBuilt.Artifacts.*.tar.gz and installing the version of dotnet referenced in global.json"
echo ""
}

positional_args=()
while :; do
if [ $# -le 0 ]; then
break
fi
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
case $lowerI in
"-?"|-h|--help)
usage
exit 0
;;
*)
positional_args+=("$1")
;;
esac

shift
done

# Check for the archive text file which describes the location of the archive files to download
if [ ! -f $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt ]; then
echo " ERROR: $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt does not exist. Cannot determine which archives to download. Exiting..."
exit -1
fi

downloadRefPkgs=true
downloadArtifacts=true
installDotnet=true

# Check to make sure curl exists to download the archive files
if ! command -v curl &> /dev/null
then
echo " ERROR: curl not found. Exiting..."
exit -1
fi

# Check if the reference packages archive exists
if [ -f $SCRIPT_ROOT/packages/archive/Private.SourceBuild.ReferencePackages.*.tar.gz ]; then
echo " Private.SourceBuild.ReferencePackages.*.tar.gz exists...it will not be downloaded"
downloadRefPkgs=false
fi

# Check if Private.SourceBuilt artifacts archive exists
if [ -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts.*.tar.gz ]; then
echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded"
downloadArtifacts=false
fi

# Check if dotnet is installed
if [ -d $SCRIPT_ROOT/.dotnet ]; then
echo " ./.dotnet SDK directory exists...it will not be installed"
installDotnet=false;
fi

# Read the archive text file to get the archives to download and download them
while read -r line; do
if [[ $line == *"Private.SourceBuild.ReferencePackages"* ]]; then
if [ "$downloadRefPkgs" == "true" ]; then
echo " Downloading ref pkgs..."
(cd $SCRIPT_ROOT/packages/archive/ && curl -O $line)
fi
fi
if [[ $line == *"Private.SourceBuilt.Artifacts"* ]]; then
if [ "$downloadArtifacts" == "true" ]; then
echo " Downloading source-built artifacts..."
(cd $SCRIPT_ROOT/packages/archive/ && curl -O $line)
fi
fi
done < $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt

# Check for the version of dotnet to install
if [ "$installDotnet" == "true" ]; then
echo " Installing dotnet..."
(source ./eng/common/tools.sh && InitializeDotNetCli true)
fi

0 comments on commit 5dcc817

Please sign in to comment.