diff --git a/.expeditor/scripts/verify/test_install_script.ps1 b/.expeditor/scripts/verify/test_install_script.ps1 new file mode 100644 index 0000000000..17dd31bceb --- /dev/null +++ b/.expeditor/scripts/verify/test_install_script.ps1 @@ -0,0 +1,8 @@ +Write-Host "--- Installing pester" +hab pkg install core/pester --channel stable +Import-Module "$(hab pkg path core/pester)\module\pester.psd1" +$env:HAB_NOCOLORING = "true" + +Write-Host "--- Running install.ps1 tests" +Invoke-Pester components/hab/tests/test_install_script.ps1 -EnableExit + diff --git a/.expeditor/scripts/verify/test_install_script.sh b/.expeditor/scripts/verify/test_install_script.sh new file mode 100755 index 0000000000..b517fda9d5 --- /dev/null +++ b/.expeditor/scripts/verify/test_install_script.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +if ! command -v bats >/dev/null; then + if [ "$(uname)" = "Darwin" ]; then + echo "--- Installing bats" + brew install bats-core + fi +fi + +if ! command -v gpg >/dev/null; then + if [ "$(uname)" = "Darwin" ]; then + echo "--- Installing gpg" + brew install gnupg + fi +fi + +echo "--- Testing install.sh" +# Bats in chefes/buildkite is a hab-binliked install to the default directory +# of /bin, but /bin isn't on our path. +bats components/hab/tests/test_install_script.bats diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index ca18b26650..c7ddd05bb2 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -291,6 +291,32 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: install script" + env: + HAB_LICENSE: "accept-no-persist" + command: + - .expeditor/scripts/verify/test_install_script.sh + expeditor: + executor: + docker: + timeout_in_minutes: 5 + retry: + automatic: + limit: 1 + ############################################################### + # MacOS tests + - label: "[unit] :darwin: install script" + command: + - .expeditor/scripts/verify/test_install_script.sh + expeditor: + executor: + macos: + inherit-environment-vars: true + timeout_in_minutes: 5 + retry: + automatic: + limit: 1 + ################################################################# # See "[unit] :linux: ignored"; the same approach applies here for windows @@ -527,6 +553,21 @@ steps: automatic: limit: 1 + - label: "[unit] :windows: install script" + env: + HAB_LICENSE: "accept-no-persist" + command: + - .expeditor/scripts/verify/test_install_script.ps1 + expeditor: + executor: + docker: + host_os: windows + shell: [ "powershell", "-Command" ] + timeout_in_minutes: 5 + retry: + automatic: + limit: 1 + ####################################################################### # Things that have no tests but should be built to make sure they # still build. - Linux diff --git a/components/hab/install.ps1 b/components/hab/install.ps1 index d5e506e623..3d6a4e32d4 100644 --- a/components/hab/install.ps1 +++ b/components/hab/install.ps1 @@ -23,8 +23,10 @@ param ( $ErrorActionPreference="stop" -Function Get-Version($version, $channel) { - $jsonFile = Join-Path (Get-WorkDir) "version.json" +Set-Variable packagesChefioRootUrl -Option ReadOnly -value "https://packages.chef.io/files" + +Function Get-BintrayVersion($version, $channel) { + $jsonFile = Join-Path ($workdir) "version.json" # bintray expects a '-' to separate version and release and not '/' $version = $version.Replace("/", "-") @@ -78,17 +80,48 @@ Function Get-File($url, $dst) { } Function Get-WorkDir { - Join-Path $env:temp "hab.XXXX" + $parent = [System.IO.Path]::GetTempPath() + [string] $name = [System.Guid]::NewGuid() + New-Item -ItemType Directory -Path (Join-Path $parent $name) } -Function Get-Archive($channel, $version) { +# Downloads the requested archive from packages.chef.io +Function Get-PackagesChefioArchive($channel, $version) { + $url = $packagesChefioRootUrl + if(!$version -Or $version -eq "latest") { + $hab_url="$url/$channel/habitat/latest/hab-x86_64-windows.zip" + } else { + $hab_url="$url/habitat/${version}/hab-x86_64-windows.zip" + } + $sha_url="$hab_url.sha256sum" + $hab_dest = (Join-Path ($workdir) "hab.zip") + $sha_dest = (Join-Path ($workdir) "hab.zip.shasum256") + + Get-File $hab_url $hab_dest + $result = @{ "zip" = $hab_dest } + + # Note that this will fail on versions less than 0.71.0 + # when we did not upload shasum files to bintray. + # NOTE: This is left in place because, while we don't ship <0.71.0 + # from s3 today, the intent is to move old releases over + try { + Get-File $sha_url $sha_dest + $result["shasum"] = (Get-Content $sha_dest).Split()[0] + } + catch { + Write-Warning "No shasum exists for $version. Skipping validation." + } + $result +} + +Function Get-BintrayArchive($channel, $version) { $url = "https://api.bintray.com/content/habitat/$channel/windows/x86_64/hab-$version-x86_64-windows.zip" $query = "?bt_package=hab-x86_64-windows" $hab_url="$url$query" $sha_url="$url.sha256sum$query" - $hab_dest = (Join-Path (Get-WorkDir) "hab.zip") - $sha_dest = (Join-Path (Get-WorkDir) "hab.zip.shasum256") + $hab_dest = (Join-Path ($workdir) "hab.zip") + $sha_dest = (Join-Path ($workdir) "hab.zip.shasum256") Get-File $hab_url $hab_dest $result = @{ "zip" = $hab_dest } @@ -139,7 +172,7 @@ Function Install-Habitat { $habPath = Join-Path $env:ProgramData Habitat if(Test-Path $habPath) { Remove-Item $habPath -Recurse -Force } New-Item $habPath -ItemType Directory | Out-Null - $folder = (Get-ChildItem (Join-Path (Get-WorkDir) "hab-*")) + $folder = (Get-ChildItem (Join-Path ($workdir) "hab-*")) Copy-Item "$($folder.FullName)\*" $habPath $env:PATH = New-PathString -StartingPath $env:PATH -Path $habPath $machinePath = [System.Environment]::GetEnvironmentVariable("PATH", "Machine") @@ -166,7 +199,7 @@ Function New-PathString([string]$StartingPath, [string]$Path) { } Function Expand-Zip($zipPath) { - $dest = Get-WorkDir + $dest = $workdir try { # Works on .Net 4.5 and up (as well as .Net Core) # Yes on PS v5 and up we have Expand-Archive but this works on PS v4 too @@ -203,13 +236,28 @@ Function Assert-Habitat($ident) { } } +Function Test-UsePackagesChefio($version) { + # The $_patch may contain the /release string as well. + # This is fine because we only care about major/minor for this + # comparison. + + $_major,$_minor,$_patch = $version -split ".",3,"SimpleMatch" + $v1 = New-Object -TypeName Version -ArgumentList $_major,$_minor + $v2 = New-Object -TypeName Version -ArgumentList "0.89" + !$version -Or ($v1 -ge $v2) +} + Write-Host "Installing Habitat 'hab' program" $workdir = Get-WorkDir New-Item $workdir -ItemType Directory -Force | Out-Null try { - $Version = Get-Version $Version $Channel - $archive = Get-Archive $channel $version + if(Test-UsePackagesChefio($Version)) { + $archive = Get-PackagesChefioArchive $channel $version + } else { + $Version = Get-BintrayVersion $Version $Channel + $archive = Get-BintrayArchive $channel $version + } if($archive.shasum) { Assert-Shasum $archive } diff --git a/components/hab/install.sh b/components/hab/install.sh index 3481a9d0d5..2d75129f08 100755 --- a/components/hab/install.sh +++ b/components/hab/install.sh @@ -7,7 +7,7 @@ if [ -n "${DEBUG:-}" ]; then set -x; fi BT_ROOT="https://api.bintray.com/content/habitat" BT_SEARCH="https://api.bintray.com/packages/habitat" - +readonly pcio_root="https://packages.chef.io/files" export HAB_LICENSE="accept-no-persist" main() { @@ -44,8 +44,12 @@ main() { create_workdir get_platform validate_target - get_version - download_archive + if use_packages_chef_io "$version"; then + download_packages_chef_io_archive "$version" "$channel" "$target" + else + get_bintray_version + download_bintray_archive + fi verify_archive extract_archive install_hab @@ -144,7 +148,29 @@ get_platform() { fi } -get_version() { +use_packages_chef_io() { + need_cmd cut + + local version + version="${1:-latest}" + + if [ "$version" == "latest" ]; then + info "No version specified, using packages.chef.io" + return 0 + else + local major + local minor + major="$(echo "${version}" | cut -d'.' -f1)" + minor="$(echo "${version}" | cut -d'.' -f2)" + if [ "$major" -ge 1 ] || [ "$minor" -ge 89 ]; then + info "Specified recent version >= 0.89, using packages.chef.io" + return 0 + fi + fi + return 1 +} + +get_bintray_version() { need_cmd grep need_cmd head need_cmd sed @@ -202,7 +228,41 @@ validate_target() { fi } -download_archive() { +download_packages_chef_io_archive() { + need_cmd mv + + local -r _version="${1:-latest}" + local -r _channel="${2:?}" + local -r _target="${3:?}" + local url + + if [ "$_version" == "latest" ]; then + url="${pcio_root}/${_channel}/habitat/latest/hab-${_target}.${ext}" + else + url="${pcio_root}/habitat/${_version}/hab-${_target}.${ext}" + fi + + dl_file "${url}" "${workdir}/hab-${_version}.${ext}" + dl_file "${url}.sha256sum" "${workdir}/hab-${_version}.${ext}.sha256sum" + + archive="hab-${_target}.${ext}" + sha_file="hab-${_target}.${ext}.sha256sum" + + mv -v "${workdir}/hab-${_version}.${ext}" "${archive}" + mv -v "${workdir}/hab-${_version}.${ext}.sha256sum" "${sha_file}" + + if command -v gpg >/dev/null; then + info "GnuPG tooling found, downloading signatures" + sha_sig_file="${archive}.sha256sum.asc" + key_file="${workdir}/chef.asc" + local _key_url="https://packages.chef.io/chef.asc" + + dl_file "${url}.sha256sum.asc" "${sha_sig_file}" + dl_file "${_key_url}" "${key_file}" + fi +} + +download_bintray_archive() { need_cmd cut need_cmd mv @@ -221,22 +281,26 @@ download_archive() { info "Renaming downloaded archive files" mv -v "${workdir}/hab-latest.${ext}" "${archive}" mv -v "${workdir}/hab-latest.${ext}.sha256sum" "${archive}.sha256sum" + + if command -v gpg >/dev/null; then + info "GnuPG tooling found, downloading signatures" + local _sha_sig_url="${url}.sha256sum.asc${query}" + local _key_url="https://bintray.com/user/downloadSubjectPublicKey?username=habitat" + sha_sig_file="${archive}.sha256sum.asc" + key_file="${workdir}/habitat.asc" + + dl_file "${_sha_sig_url}" "${sha_sig_file}" + dl_file "${_key_url}" "${key_file}" + fi } verify_archive() { if command -v gpg >/dev/null; then info "GnuPG tooling found, verifying the shasum digest is properly signed" - local _sha_sig_url="${url}.sha256sum.asc${query}" - local _sha_sig_file="${archive}.sha256sum.asc" - local _key_url="https://bintray.com/user/downloadSubjectPublicKey?username=habitat" - local _key_file="${workdir}/habitat.asc" - - dl_file "${_sha_sig_url}" "${_sha_sig_file}" - dl_file "${_key_url}" "${_key_file}" - gpg --no-permission-warning --dearmor "${_key_file}" + gpg --no-permission-warning --dearmor "${key_file}" gpg --no-permission-warning \ - --keyring "${_key_file}.gpg" --verify "${_sha_sig_file}" + --keyring "${key_file}.gpg" --verify "${sha_sig_file}" fi info "Verifying the shasum digest matches the downloaded archive" @@ -252,14 +316,17 @@ extract_archive() { need_cmd zcat need_cmd tar - zcat "${archive}" | tar x -C "${workdir}" archive_dir="${archive%.tar.gz}" + mkdir "${archive_dir}" + zcat "${archive}" | tar --extract --directory "${archive_dir}" --strip-components=1 + ;; zip) need_cmd unzip - unzip "${archive}" -d "${workdir}" archive_dir="${archive%.zip}" + # -j "junk paths" Strips leading paths from files, + unzip -j "${archive}" -d "${archive_dir}" ;; *) exit_with "Unrecognized file extension when extracting: ${ext}" 4 diff --git a/components/hab/tests/test_install_script.bats b/components/hab/tests/test_install_script.bats new file mode 100644 index 0000000000..ed025b462d --- /dev/null +++ b/components/hab/tests/test_install_script.bats @@ -0,0 +1,104 @@ +setup() { + if [ -n "$CI" ]; then + rm -f /bin/hab + rm -rf /hab/pkgs/core/hab + else + echo "Not running in CI, skipping cleanup" + fi +} + +darwin() { + [ "$(uname)" == "Darwin" ] +} + +linux() { + [ "$(uname)" == "Linux" ] +} + +installed_version() { + hab --version | cut -d'/' -f1 +} + +installed_target() { + version_release="$(hab --version | cut -d' ' -f2)" + version="$(cut -d'/' -f1 <<< "$version_release")" + release="$(cut -d'/' -f2 <<< "$version_release")" + cat /hab/pkgs/core/hab/"$version"/"$release"/TARGET +} + +@test "Install latest for x86_86-linux" { + linux || skip "Did not detect a Linux system" + run components/hab/install.sh + + [ "$status" -eq 0 ] + [ "$(installed_target)" == "x86_64-linux" ] +} + +@test "Install specific version for x86_64-linux" { + linux || skip "Did not detect a Linux system" + run components/hab/install.sh -v 0.90.6 + + [ "$status" -eq 0 ] + [ "$(installed_version)" == "hab 0.90.6" ] + [ "$(installed_target)" == "x86_64-linux" ] +} + +@test "Install from bintray for x86_84-linux" { + linux || skip "Did not detect a Linux system" + run components/hab/install.sh -v 0.79.1 + + [ "$status" -eq 0 ] + [ "$(installed_version)" == "hab 0.79.1" ] + [ "$(installed_target)" == "x86_64-linux" ] +} + +@test "Install latest for x86_64-linux-kernel2" { + linux || skip "Did not detect a Linux system" + run components/hab/install.sh -t "x86_64-linux-kernel2" + + [ "$status" -eq 0 ] + [ "$(installed_target)" == "x86_64-linux-kernel2" ] +} + +@test "Install specific version for x86_64-linux-kernel2" { + linux || skip "Did not detect a Linux system" + run components/hab/install.sh -v 0.90.6 -t "x86_64-linux-kernel2" + + [ "$status" -eq 0 ] + [ "$(installed_version)" == "hab 0.90.6" ] + [ "$(installed_target)" == "x86_64-linux-kernel2" ] +} + +@test "Install from bintray for x86_84-linux-kernel2" { + linux || skip "Did not detect a Linux system" + run components/hab/install.sh -v 0.79.1 -t "x86_64-linux-kernel2" + + [ "$status" -eq 0 ] + [ "$(installed_version)" == "hab 0.79.1" ] + [ "$(installed_target)" == "x86_64-linux-kernel2" ] +} + +@test "Install latest for x86_86-darwin" { + darwin || skip "Did not detect a Darwin system" + run components/hab/install.sh + + [ "$status" -eq 0 ] +} + +@test "Install specific version for x86_64-darwin" { + darwin || skip "Did not detect a Darwin system" + run components/hab/install.sh -v 0.90.6 + + [ "$status" -eq 0 ] + [ "$(installed_version)" == "hab 0.90.6" ] +} + +@test "Install from bintray for x86_84-darwin" { + darwin || skip "Did not detect a Darwin system" + run components/hab/install.sh -v 0.79.1 + + [ "$status" -eq 0 ] + [ "$(installed_version)" == "hab 0.79.1" ] +} + + diff --git a/components/hab/tests/test_install_script.ps1 b/components/hab/tests/test_install_script.ps1 new file mode 100644 index 0000000000..156e2535f2 --- /dev/null +++ b/components/hab/tests/test_install_script.ps1 @@ -0,0 +1,23 @@ +Describe "Install habitat using install.ps1" { + It "can install the latest version of Habitat" { + components/hab/install.ps1 + $LASTEXITCODE | Should -Be 0 + (Get-Command hab).Path | Should -Be "C:\ProgramData\Habitat\hab.exe" + } + + It "can install a specific version of Habitat" { + components/hab/install.ps1 -v 0.90.6 + $LASTEXITCODE | Should -Be 0 + + $result = hab --version + $result | Should -Match "hab 0.90.6/*" + } + + It "can install a specific version of Habitat from Bintray" { + components/hab/install.ps1 -v 0.79.1 + $LASTEXITCODE | Should -Be 0 + + $result = hab --version + $result | Should -Match "hab 0.79.1/*" + } +}