Merge pull request #7119 from habitat-sh/sm/update-install-scripts

Update install scripts to pull from packages.chef.io for new releases

.expeditor/scripts/verify/test_install_script.ps1

@@ -0,0 +1,8 @@
+Write-Host "--- Installing pester"
+hab pkg install core/pester --channel stable
+Import-Module "$(hab pkg path core/pester)\module\pester.psd1"
+$env:HAB_NOCOLORING = "true"
+
+Write-Host "--- Running install.ps1 tests"
+Invoke-Pester components/hab/tests/test_install_script.ps1 -EnableExit
+

.expeditor/scripts/verify/test_install_script.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+if ! command -v bats >/dev/null; then
+  if [ "$(uname)" = "Darwin" ]; then
+    echo "--- Installing bats"
+    brew install bats-core
+  fi
+fi
+
+if ! command -v gpg >/dev/null; then
+  if [ "$(uname)" = "Darwin" ]; then
+    echo "--- Installing gpg"
+    brew install gnupg
+  fi
+fi
+
+echo "--- Testing install.sh"
+# Bats in chefes/buildkite is a hab-binliked install to the default directory
+# of /bin, but /bin isn't on our path. 
+bats components/hab/tests/test_install_script.bats

.expeditor/verify.pipeline.yml

@@ -291,6 +291,32 @@ steps:
       automatic:
         limit: 1
 
+  - label: "[unit] :linux: install script"
+    env:
+      HAB_LICENSE: "accept-no-persist"
+    command:
+      - .expeditor/scripts/verify/test_install_script.sh
+    expeditor:
+      executor:
+        docker:
+    timeout_in_minutes: 5
+    retry:
+      automatic:
+        limit: 1
+  ###############################################################
+  # MacOS tests
+  - label: "[unit] :darwin: install script"
+    command:
+      - .expeditor/scripts/verify/test_install_script.sh
+    expeditor:
+      executor:
+        macos:
+          inherit-environment-vars: true
+    timeout_in_minutes: 5
+    retry:
+      automatic:
+        limit: 1
+
  #################################################################
 
   # See "[unit] :linux: ignored"; the same approach applies here for windows
@@ -527,6 +553,21 @@ steps:
       automatic:
         limit: 1
 
+  - label: "[unit] :windows: install script"
+    env:
+      HAB_LICENSE: "accept-no-persist"
+    command:
+      - .expeditor/scripts/verify/test_install_script.ps1
+    expeditor:
+      executor:
+        docker: 
+          host_os: windows
+          shell: [ "powershell", "-Command" ]
+    timeout_in_minutes: 5
+    retry:
+      automatic:
+        limit: 1
+
 #######################################################################
 # Things that have no tests but should be built to make sure they
 # still build. - Linux

components/hab/install.ps1

@@ -23,8 +23,10 @@ param (
 
 $ErrorActionPreference="stop"
 
-Function Get-Version($version, $channel) {
-    $jsonFile = Join-Path (Get-WorkDir) "version.json"
+Set-Variable packagesChefioRootUrl -Option ReadOnly -value "https://packages.chef.io/files"
+
+Function Get-BintrayVersion($version, $channel) {
+    $jsonFile = Join-Path ($workdir) "version.json"
 
     # bintray expects a '-' to separate version and release and not '/'
     $version = $version.Replace("/", "-")
@@ -78,17 +80,48 @@ Function Get-File($url, $dst) {
 }
 
 Function Get-WorkDir {
-    Join-Path $env:temp "hab.XXXX"
+  $parent = [System.IO.Path]::GetTempPath()
+  [string] $name = [System.Guid]::NewGuid()
+  New-Item -ItemType Directory -Path (Join-Path $parent $name)
 }
 
-Function Get-Archive($channel, $version) {
+# Downloads the requested archive from packages.chef.io
+Function Get-PackagesChefioArchive($channel, $version) {
+    $url = $packagesChefioRootUrl
+    if(!$version -Or $version -eq "latest") {
+      $hab_url="$url/$channel/habitat/latest/hab-x86_64-windows.zip"
+    } else {
+      $hab_url="$url/habitat/${version}/hab-x86_64-windows.zip"
+    }
+    $sha_url="$hab_url.sha256sum"
+    $hab_dest = (Join-Path ($workdir) "hab.zip")
+    $sha_dest = (Join-Path ($workdir) "hab.zip.shasum256")
+
+    Get-File $hab_url $hab_dest
+    $result = @{ "zip" = $hab_dest }
+
+    # Note that this will fail on versions less than 0.71.0
+    # when we did not upload shasum files to bintray.
+    # NOTE: This is left in place because, while we don't ship <0.71.0
+    # from s3 today, the intent is to move old releases over 
+    try {
+        Get-File $sha_url $sha_dest
+        $result["shasum"] = (Get-Content $sha_dest).Split()[0]
+    }
+    catch {
+        Write-Warning "No shasum exists for $version. Skipping validation."
+    }
+    $result
+}
+
+Function Get-BintrayArchive($channel, $version) {
     $url = "https://api.bintray.com/content/habitat/$channel/windows/x86_64/hab-$version-x86_64-windows.zip"
     $query = "?bt_package=hab-x86_64-windows"
 
     $hab_url="$url$query"
     $sha_url="$url.sha256sum$query"
-    $hab_dest = (Join-Path (Get-WorkDir) "hab.zip")
-    $sha_dest = (Join-Path (Get-WorkDir) "hab.zip.shasum256")
+    $hab_dest = (Join-Path ($workdir) "hab.zip")
+    $sha_dest = (Join-Path ($workdir) "hab.zip.shasum256")
 
     Get-File $hab_url $hab_dest
     $result = @{ "zip" = $hab_dest }
@@ -139,7 +172,7 @@ Function Install-Habitat {
     $habPath = Join-Path $env:ProgramData Habitat
     if(Test-Path $habPath) { Remove-Item $habPath -Recurse -Force }
     New-Item $habPath -ItemType Directory | Out-Null
-    $folder = (Get-ChildItem (Join-Path (Get-WorkDir) "hab-*"))
+    $folder = (Get-ChildItem (Join-Path ($workdir) "hab-*"))
     Copy-Item "$($folder.FullName)\*" $habPath
     $env:PATH = New-PathString -StartingPath $env:PATH -Path $habPath
     $machinePath = [System.Environment]::GetEnvironmentVariable("PATH", "Machine")
@@ -166,7 +199,7 @@ Function New-PathString([string]$StartingPath, [string]$Path) {
 }
 
 Function Expand-Zip($zipPath) {
-    $dest = Get-WorkDir
+    $dest = $workdir
     try {
         # Works on .Net 4.5 and up (as well as .Net Core)
         # Yes on PS v5 and up we have Expand-Archive but this works on PS v4 too
@@ -203,13 +236,28 @@ Function Assert-Habitat($ident) {
     }
 }
 
+Function Test-UsePackagesChefio($version) {
+    # The $_patch may contain the /release string as well.
+    # This is fine because we only care about major/minor for this 
+    # comparison. 
+
+    $_major,$_minor,$_patch = $version -split ".",3,"SimpleMatch"
+    $v1 = New-Object -TypeName Version -ArgumentList $_major,$_minor
+    $v2 = New-Object -TypeName Version -ArgumentList "0.89"
+    !$version -Or ($v1 -ge $v2)
+}
+
 Write-Host "Installing Habitat 'hab' program"
 
 $workdir = Get-WorkDir
 New-Item $workdir -ItemType Directory -Force | Out-Null
 try {
-    $Version = Get-Version $Version $Channel
-    $archive = Get-Archive $channel $version
+    if(Test-UsePackagesChefio($Version)) { 
+      $archive = Get-PackagesChefioArchive $channel $version
+    } else {
+      $Version = Get-BintrayVersion $Version $Channel
+      $archive = Get-BintrayArchive $channel $version
+    }
     if($archive.shasum) {
         Assert-Shasum $archive
     }

components/hab/install.sh

@@ -7,7 +7,7 @@ if [ -n "${DEBUG:-}" ]; then set -x; fi
 
 BT_ROOT="https://api.bintray.com/content/habitat"
 BT_SEARCH="https://api.bintray.com/packages/habitat"
-
+readonly pcio_root="https://packages.chef.io/files"
 export HAB_LICENSE="accept-no-persist"
 
 main() {
@@ -44,8 +44,12 @@ main() {
   create_workdir
   get_platform
   validate_target
-  get_version
-  download_archive
+  if use_packages_chef_io "$version"; then
+    download_packages_chef_io_archive "$version" "$channel" "$target"
+  else
+    get_bintray_version
+    download_bintray_archive
+  fi
   verify_archive
   extract_archive
   install_hab
@@ -144,7 +148,29 @@ get_platform() {
   fi
 }
 
-get_version() {
+use_packages_chef_io() {
+  need_cmd cut
+
+  local version
+  version="${1:-latest}"
+
+  if [ "$version" == "latest" ]; then
+    info "No version specified, using packages.chef.io"
+    return 0
+  else 
+    local major 
+    local minor
+    major="$(echo "${version}" | cut -d'.' -f1)"
+    minor="$(echo "${version}" | cut -d'.' -f2)"
+    if [ "$major" -ge 1 ] || [ "$minor" -ge 89 ]; then
+      info "Specified recent version >= 0.89, using packages.chef.io"
+      return 0
+    fi
+  fi
+  return 1
+}
+
+get_bintray_version() {
   need_cmd grep
   need_cmd head
   need_cmd sed
@@ -202,7 +228,41 @@ validate_target() {
   fi
 }
 
-download_archive() {
+download_packages_chef_io_archive() {
+  need_cmd mv
+
+  local -r _version="${1:-latest}"
+  local -r _channel="${2:?}"
+  local -r _target="${3:?}"
+  local url
+
+  if [ "$_version" == "latest" ]; then
+    url="${pcio_root}/${_channel}/habitat/latest/hab-${_target}.${ext}"
+  else 
+    url="${pcio_root}/habitat/${_version}/hab-${_target}.${ext}"
+  fi
+
+  dl_file "${url}" "${workdir}/hab-${_version}.${ext}"
+  dl_file "${url}.sha256sum" "${workdir}/hab-${_version}.${ext}.sha256sum"
+
+  archive="hab-${_target}.${ext}"
+  sha_file="hab-${_target}.${ext}.sha256sum"
+
+  mv -v "${workdir}/hab-${_version}.${ext}" "${archive}"
+  mv -v "${workdir}/hab-${_version}.${ext}.sha256sum" "${sha_file}"
+
+  if command -v gpg >/dev/null; then
+    info "GnuPG tooling found, downloading signatures"
+    sha_sig_file="${archive}.sha256sum.asc"
+    key_file="${workdir}/chef.asc"
+    local _key_url="https://packages.chef.io/chef.asc"
+
+    dl_file "${url}.sha256sum.asc" "${sha_sig_file}"
+    dl_file "${_key_url}" "${key_file}" 
+  fi
+}
+
+download_bintray_archive() {
   need_cmd cut
   need_cmd mv
 
@@ -221,22 +281,26 @@ download_archive() {
   info "Renaming downloaded archive files"
   mv -v "${workdir}/hab-latest.${ext}" "${archive}"
   mv -v "${workdir}/hab-latest.${ext}.sha256sum" "${archive}.sha256sum"
+
+  if command -v gpg >/dev/null; then
+    info "GnuPG tooling found, downloading signatures"
+    local _sha_sig_url="${url}.sha256sum.asc${query}"
+    local _key_url="https://bintray.com/user/downloadSubjectPublicKey?username=habitat"
+    sha_sig_file="${archive}.sha256sum.asc"
+    key_file="${workdir}/habitat.asc"
+
+    dl_file "${_sha_sig_url}" "${sha_sig_file}"
+    dl_file "${_key_url}" "${key_file}" 
+  fi
 }
 
 verify_archive() {
   if command -v gpg >/dev/null; then
     info "GnuPG tooling found, verifying the shasum digest is properly signed"
-    local _sha_sig_url="${url}.sha256sum.asc${query}"
-    local _sha_sig_file="${archive}.sha256sum.asc"
-    local _key_url="https://bintray.com/user/downloadSubjectPublicKey?username=habitat"
-    local _key_file="${workdir}/habitat.asc"
-
-    dl_file "${_sha_sig_url}" "${_sha_sig_file}"
-    dl_file "${_key_url}" "${_key_file}"
 
-    gpg --no-permission-warning --dearmor "${_key_file}"
+    gpg --no-permission-warning --dearmor "${key_file}"
     gpg --no-permission-warning \
-      --keyring "${_key_file}.gpg" --verify "${_sha_sig_file}"
+      --keyring "${key_file}.gpg" --verify "${sha_sig_file}"
   fi
 
   info "Verifying the shasum digest matches the downloaded archive"
@@ -252,14 +316,17 @@ extract_archive() {
       need_cmd zcat
       need_cmd tar
 
-      zcat "${archive}" | tar x -C "${workdir}"
       archive_dir="${archive%.tar.gz}"
+      mkdir "${archive_dir}"
+      zcat "${archive}" | tar --extract --directory "${archive_dir}" --strip-components=1
+
       ;;
     zip)
       need_cmd unzip
 
-      unzip "${archive}" -d "${workdir}"
       archive_dir="${archive%.zip}"
+      # -j "junk paths" Strips leading paths from files,
+      unzip -j "${archive}" -d "${archive_dir}"
       ;;
     *)
       exit_with "Unrecognized file extension when extracting: ${ext}" 4