buildLoadup.yml

#*******************************************************************************
# buidLoadup.yml
#
# Interlisp workflow to build Medley release and push it to github. This workflow
# is platform independent - but runs on Linux/amd64.
#
# This workflow contains a sentry that causes it to skip the build (as identified
# by its commit SHA) if its already been done.  Setting the "force" input to true
# will bypass this sentry,
#
# 2022-01-17 Frank Halasz based on an earlier version of buildLoadup for Medley.
#
# Copyright 2022-2023 by Interlisp.org
#
# ******************************************************************************

name: Build/Push Medley Release

# Run this workflow on ...
on:
  workflow_dispatch:
    inputs:
      draft:
        description: "Mark this as a draft release"
        type: choice
        options:
        - 'false'
        - 'true'
      force:
        description: "Force build even if build already successfully completed for this commit"
        type: choice
        options:
        - 'false'
        - 'true'

  workflow_call:
    outputs:
      successful:
        description: "'True' if medley build completed successully"
        value: ${{ jobs.complete.outputs.build_successful }}
    inputs:
      draft:
        description: "Mark this as a draft release"
        required: false
        type: string
        default: 'false'
      force:
        description: "Force build even if build already successfully completed for this commit"
        required: false
        type: string
        default: 'false'
    secrets:
      OIO_SSH_KEY:
        required: true

defaults:
  run:
    shell: bash


jobs:

# JOB: inputs #######################################################################

  # Regularize the inputs so they can be referenced the same way whether they are
  # the result of a workflow_dispatch or a workflow_call

  inputs:
    runs-on: ubuntu-latest
    outputs:
      draft: ${{ steps.one.outputs.draft }}
      force: ${{ steps.one.outputs.force }}
    steps:
      - id: one
        run: >
          if [ '${{ toJSON(inputs) }}' = 'null'  ];
          then
              echo "workflow_dispatch";
              echo "draft=${{ github.event.inputs.draft }}" >> $GITHUB_OUTPUT;
              echo "force=${{ github.event.inputs.force }}" >> $GITHUB_OUTPUT;
          else
              echo "workflow_call";
              echo "draft=${{ inputs.draft }}" >> $GITHUB_OUTPUT;
              echo "force=${{ inputs.force }}" >> $GITHUB_OUTPUT;
          fi


# JOB: sentry #######################################################################

  # Use sentry-action to determine if this release has already been built
  # based on the latest commit to the repo

  sentry:
    needs: inputs
    runs-on: ubuntu-latest
    outputs:
      release_not_built: ${{ steps.check.outputs.release_not_built }}

    steps:
      # Checkout the actions for this repo owner
      - name: Checkout Actions
        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/.github
          path: ./Actions_${{ github.sha }}
      - run: mv ./Actions_${{ github.sha }}/actions ../actions && rm -rf ./Actions_${{ github.sha }}

      # Check if build already run for this commit
      - name: Build already completed?
        id: check
        continue-on-error: true
        uses: ./../actions/check-sentry-action
        with:
          tag: "loadup"


# JOB: loadup #######################################################################

  #
  # Do the loadup and push to release on github
  # 

  loadup:

    runs-on: ubuntu-latest

    outputs:
      combined_release_tag: ${{ steps.job_outputs.outputs.COMBINED_RELEASE_TAG }}
      medley_release_tag: ${{ steps.job_outputs.outputs.MEDLEY_RELEASE_TAG }}
      medley_short_release_tag: ${{ steps.job_outputs.outputs.MEDLEY_SHORT_RELEASE_TAG }}
      artifacts_filename_template: ${{ steps.job_outputs.outputs.ARTIFACTS_FILENAME_TEMPLATE }}
      release_url: ${{ steps.push.outputs.html_url }}

    needs: [inputs, sentry]
    if: |
      needs.sentry.outputs.release_not_built == 'true'
      || needs.inputs.outputs.force == 'true'

    steps:
      # Checkout the actions for this repo owner
      - name: Checkout Actions
        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/.github
          path: ./Actions_${{ github.sha }}
      - run: mv ./Actions_${{ github.sha }}/actions ../actions && rm -rf ./Actions_${{ github.sha }}

      # Checkout latest commit
      - name: Checkout Medley
        uses: actions/checkout@v3

      # Setup release tag
      - name: Setup Release Tag
        id: tag
        uses: ./../actions/release-tag-action

      # Get Maiko release information, retrieves the name of the latest
      # release.  Used to download the correct Maiko release
      # Find latest release (draft or normal)
      - name: Get maiko release information
        id: maiko
        run: |
          tag=""
          if [ "${{ needs.inputs.outputs.draft }}" = "true" ];
          then
            gh release list --repo ${{ github.repository_owner }}/maiko | grep Draft >/tmp/releases-$$
            if [ $? -eq 0 ];
            then
              tag=$(head -n 1 /tmp/releases-$$ | awk '{ print $3 }')
            fi
          fi
          if [ -z "${tag}" ];
          then
            tag=$(gh release list --repo ${{ github.repository_owner }}/maiko | grep Latest | head -n 1 | awk '{ print $3 }')
          fi
          echo "maiko_tag=${tag}" >> ${GITHUB_OUTPUT}
        env:
          GITHUB_TOKEN: ${{ secrets.MAIKO_TOKEN }}

      # Setup environment variables & establish job outputs
      - name: Setup Environment Variables
        run: |
          echo "build_time=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> ${GITHUB_OUTPUT}
          echo "TARBALL_DIR=/tmp/tarballs" >>${GITHUB_ENV}
          echo "MEDLEY_RELEASE_TAG=${RELEASE_TAG}" >>${GITHUB_ENV}
          echo "MAIKO_RELEASE_TAG=${{ steps.maiko.outputs.maiko_tag }}" >>${GITHUB_ENV}
          echo "ARTIFACTS_FILENAME_TEMPLATE=medley-full-@@PLATFORM@@-@@ARCH@@-@@MEDLEY.RELEASE@@_@@MAIKO.RELEASE@@" >>${GITHUB_ENV}
      - name: More Environment Variables
        run: |
          echo "MEDLEY_SHORT_RELEASE_TAG=${MEDLEY_RELEASE_TAG#medley-}" >>${GITHUB_ENV}
          echo "MAIKO_SHORT_RELEASE_TAG=${MAIKO_RELEASE_TAG#maiko-}" >>${GITHUB_ENV}
      - name: Even More Environment Variables
        run: |
          echo "COMBINED_RELEASE_TAG=${MEDLEY_SHORT_RELEASE_TAG}_${MAIKO_SHORT_RELEASE_TAG}" >>${GITHUB_ENV}
      - name: Establish job outputs
        id: job_outputs
        run: |
          echo "COMBINED_RELEASE_TAG=${COMBINED_RELEASE_TAG}" >> ${GITHUB_OUTPUT}
          echo "MEDLEY_RELEASE_TAG=${MEDLEY_RELEASE_TAG}" >> ${GITHUB_OUTPUT}
          echo "MEDLEY_SHORT_RELEASE_TAG=${MEDLEY_SHORT_RELEASE_TAG}" >> ${GITHUB_OUTPUT}
          echo "ARTIFACTS_FILENAME_TEMPLATE=${ARTIFACTS_FILENAME_TEMPLATE}" >> ${GITHUB_OUTPUT}

      # Setup some needed dirs in workspace
      - name: Create work dirs
        run: mkdir -p ${TARBALL_DIR}

      # Download Maiko Release Assets and untar it
      - name: Download Release Assets
        run: |
          gh release download ${MAIKO_RELEASE_TAG} \
                              -D ${TARBALL_DIR}    \
                              --repo ${{ github.repository_owner }}/maiko \
                              -p '*.tgz'
          tar -xzf "${TARBALL_DIR}/${MAIKO_RELEASE_TAG}-linux.x86_64.tgz"
        env:
          GITHUB_TOKEN: ${{ secrets.MAIKO_TOKEN }}

      # Checkout Notecards and tar it in the tarballsdir
      - name: Checkout Notecards
        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/notecards
          path: ./notecards
      - name: Tar notecards into tarball dir
        run: |
          mv ./notecards ../notecards
          cd ../notecards
          git archive --format=tgz --output="${TARBALL_DIR}/notecards.tgz" --prefix=notecards/ main

      # Install vnc
      - name: Install vnc
        run: sudo apt-get update && sudo apt-get install -y tightvncserver

      - name: Build Loadup sysouts and databases
        run: |
          Xvnc -geometry 1280x720 :0 &
          export DISPLAY=":0"
          scripts/loadup-all.sh -apps
          scripts/loadup-db.sh

      - name: Build release tars
        run: |
          scripts/release-make-tars.sh "${MEDLEY_RELEASE_TAG}"
          mv releases/"${MEDLEY_SHORT_RELEASE_TAG}"/*.tgz "${TARBALL_DIR}"

      # Push the release up to github releases
      - name: Delete existing release with same tag (if any)
        uses: cb80/delrel@latest
        with:
            tag: ${{ env.MEDLEY_RELEASE_TAG }}
        continue-on-error: true

      - name: Push the release
        id: push
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: true
          artifacts:
            # ${{ env.TARBALL_DIR }}/notecards.tgz,
            ${{ env.TARBALL_DIR }}/${{ env.MEDLEY_RELEASE_TAG }}-loadups.tgz,
            ${{ env.TARBALL_DIR }}/${{ env.MEDLEY_RELEASE_TAG }}-runtime.tgz
          tag: ${{ env.MEDLEY_RELEASE_TAG }}
          draft: ${{ needs.inputs.outputs.draft }}
          prerelease: false
          generateReleaseNotes: true
          token: ${{ secrets.GITHUB_TOKEN }}

      # Save the tarball directory for subsequent jobs
      - name: Save tarballs
        uses: actions/upload-artifact@v3
        with:
          name: tarballs
          path: ${{ env.TARBALL_DIR }}
          if-no-files-found: error


# JOB: linux_installer ##############################################################

  #
  # Create the linux installers (.deb and .tgz) and push to release on github
  #
  linux_installer:

    runs-on: ubuntu-latest

    needs: [inputs, sentry, loadup]
    if: |
      needs.sentry.outputs.release_not_built == 'true'
      || needs.inputs.outputs.force == 'true'

    steps:

      # Checkout latest commit
      - name: Checkout Medley
        uses: actions/checkout@v3

      # Environment variables
      - name: Environment variables
        run: |
          echo "DEBS_DIR=installers/deb/debs" >>${GITHUB_ENV}
          echo "TARS_DIR=installers/deb/tars" >>${GITHUB_ENV}
          echo "TARBALL_DIR=installers/deb/tmp/tarballs" >>${GITHUB_ENV}
          echo "MEDLEY_RELEASE_TAG=${{ needs.loadup.outputs.medley_release_tag }}" \
                >>${GITHUB_ENV}
          echo "ARTIFACTS_FILENAME_TEMPLATE=${{ needs.loadup.outputs.artifacts_filename_template }}" >>${GITHUB_ENV}

      # Create taball dir
      - run: mkdir -p ${TARBALL_DIR}

      #  Get the tarballs
      - name: Get tarballs
        uses: actions/download-artifact@v3
        with:
          name: tarballs
          path: ${{ env.TARBALL_DIR }}

      # Build the deb files as well as the tgz files
      - name: Build .deb files for 3 architectures
        id: debs
        run: |
          cd installers/deb
          ./build_deb.sh

      # Push the debs and tgz up to github releases
      - name: Push the release
        id: push_release
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: true
          artifacts:
            ${{ env.DEBS_DIR }}/*.deb,
            ${{ env.TARS_DIR }}/*.tgz
          tag: ${{ env.MEDLEY_RELEASE_TAG }}
          token: ${{ secrets.GITHUB_TOKEN }}
          omitBodyDuringUpdate: true
          omitDraftDuringUpdate: true
          omitNameDuringUpdate: true
          omitPrereleaseDuringUpdate: true



# JOB: macos_installer ##############################################################

  #
  # Create the macos installers (.dmg and .zip) and push to release on github
  #
  macos_installer:

    runs-on: macos-12

    needs: [inputs, sentry, loadup]
    if: |
      needs.sentry.outputs.release_not_built == 'true'
      || needs.inputs.outputs.force == 'true'
#    if: false

    defaults:
      run:
        shell: bash

    steps:

      # Checkout latest commit
      - name: Checkout Medley
        uses: actions/checkout@v3

      # Environment variables
      - name: Environment variables
        run: |
          MACOS_DIR=installers/macos
          echo "MACOS_DIR=${MACOS_DIR}" >>${GITHUB_ENV}
          echo "ARTIFACTS_DIR=${MACOS_DIR}/artifacts" >>${GITHUB_ENV}
          echo "TARBALL_DIR=${MACOS_DIR}/tmp/tarballs" >>${GITHUB_ENV}
          echo "MEDLEY_RELEASE_TAG=${{ needs.loadup.outputs.medley_release_tag }}" \
                >>${GITHUB_ENV}
          echo "ARTIFACTS_FILENAME_TEMPLATE=${{ needs.loadup.outputs.artifacts_filename_template }}" >>${GITHUB_ENV}

      # Create tarball dir
      - run: mkdir -p ${TARBALL_DIR}

      #  Get the tarballs
      - name: Get tarballs
        uses: actions/download-artifact@v3
        with:
          name: tarballs
          path: ${{ env.TARBALL_DIR }}

      # Build the .dmg and .zip files
      - name: Build .dmg & .zip files
        id: dmg-zip
        run: |
          cd ${MACOS_DIR}
          ./build_artifacts.sh

      # Push the .dmg and .zip up to github releases
      - name: Push the release
        id: push_release
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: true
          artifacts:
            # 2023-07-20 not yet building dmg -- ${{ env.ARTIFACTS_DIR }}/*.dmg,
            ${{ env.ARTIFACTS_DIR }}/*.zip
          tag: ${{ env.MEDLEY_RELEASE_TAG }}
          token: ${{ secrets.GITHUB_TOKEN }}
          omitBodyDuringUpdate: true
          omitDraftDuringUpdate: true
          omitNameDuringUpdate: true
          omitPrereleaseDuringUpdate: true



# JOB: windows_installer #############################################################

  #
  # Create the Windows installer, push it up to the release on github and
  # update the downloads page on OIO
  #
  windows_installer:

    runs-on: windows-latest

    needs: [inputs, sentry, loadup]
    if: |
      needs.sentry.outputs.release_not_built == 'true'
      || needs.inputs.outputs.force == 'true'

    outputs:
      windows_installer_filename: ${{ steps.jobout.outputs.INSTALLER_FILENAME }}

    steps:

      # Checkout latest commit
      - name: Checkout Medley
        uses: actions/checkout@v3

      # Store the values output from loadup job as environment variables
      - name: Environment Variables
        shell: powershell
        run: |
          $crt="${{ needs.loadup.outputs.combined_release_tag }}"
          echo "COMBINED_RELEASE_TAG=$crt" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
          $mrt="${{ needs.loadup.outputs.medley_release_tag }}"
          echo "MEDLEY_RELEASE_TAG=$mrt" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
          $msrt="${{ needs.loadup.outputs.medley_short_release_tag }}"
          echo "MEDLEY_SHORT_RELEASE_TAG=$msrt" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
          $aft="${{ needs.loadup.outputs.artifacts_filename_template }}"
          echo "ARTIFACTS_FILENAME_TEMPLATE=$aft" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append

      # Download vnc viewer
      - name: Download vncviewer
        shell: powershell
        run: |
          $url = "https://online.interlisp.org/downloads/vncviewer64-1.12.0.exe"
          $output = "installers\win\vncviewer64-1.12.0.exe"
          (New-Object System.Net.WebClient).DownloadFile($url, $output)

      # Run iscc.exe to compile the installer
      - name: Compile medley.iss
        shell: powershell
        run: |
           iscc installers\win\medley.iss


      # Upload windows installer to release
      - name: Upload windows installer to release
        id: push
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: true
          artifacts: installers/win/medley-*.exe
          tag: ${{ env.MEDLEY_RELEASE_TAG }}
          token: ${{ secrets.GITHUB_TOKEN }}
          omitBodyDuringUpdate: true
          omitDraftDuringUpdate: true
          omitNameDuringUpdate: true
          omitPrereleaseDuringUpdate: true


# JOB: downloads_page ################################################################

  #
  # Update the downloads page on OIO
  #

  downloads_page:

    runs-on: ubuntu-latest

    needs: [inputs, sentry, loadup, linux_installer, macos_installer, windows_installer]
    if: |
      needs.sentry.outputs.release_not_built == 'true'
      || needs.inputs.outputs.force == 'true'

    steps:

      # Environment variables
      - name: Environment Varibales (from other jobs)
        run: |
          crt="${{ needs.loadup.outputs.combined_release_tag }}"
          echo "COMBINED_RELEASE_TAG=${crt}" >>${GITHUB_ENV}
          mrt="${{ needs.loadup.outputs.medley_release_tag }}"
          echo "MEDLEY_RELEASE_TAG=${mrt}" >>${GITHUB_ENV}
          msrt="${{ needs.loadup.outputs.medley_short_release_tag }}"
          echo "MEDLEY_SHORT_RELEASE_TAG=${msrt}" >>${GITHUB_ENV}

      # Checkout latest commit
      - name: Checkout Medley
        uses: actions/checkout@v3


      # Upload a dummy file to release
      # Needed since download url of the release changes on every update
      # So this will be the final update before creating downloads page
      # and we can use its url for the page
      - run: echo "placeholder" >placeholder.txt
      - name: Upload windows installer to release
        id: pushph
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: true
          artifacts: placeholder.txt
          tag: ${{ env.MEDLEY_RELEASE_TAG }}
          token: ${{ secrets.GITHUB_TOKEN }}
          omitBodyDuringUpdate: true
          omitDraftDuringUpdate: true
          omitNameDuringUpdate: true
          omitPrereleaseDuringUpdate: true

      # Update the downloads page and the man page on OIO
      - name: Update the downloads page and the man page to the OIO static page host
        shell: bash
        run: |
            # Figure out filenames
            download_url="${{ steps.pushph.outputs.html_url }}"
            download_url="${download_url/\/tag\//\/download\/}"
            local_template="installers/downloads_page/medley_downloads.html"
            local_filename="medley_downloads.html"
            local_manpath="docs/man-page/man_medley.html"
            if [ "${{ needs.inputs.outputs.draft }}" = "true" ];
            then
                remote_filename="draft_downloads"
                remote_manname="man_draft.html"
            else
                remote_filename="${local_filename%.html}"
                remote_manname="man_medley.html"
            fi
            remote_filepath="/srv/oio/static/${remote_filename}"
            remote_manpath="/srv/oio/static/${remote_manname}"
            # Fill in downloads page template
            sed \
               -e "s/@@@MEDLEY.SHORT.RELEASE.TAG@@@/${MEDLEY_SHORT_RELEASE_TAG}/g" \
               -e "s/@@@COMBINED.RELEASE.TAG@@@/${COMBINED_RELEASE_TAG}/g" \
               -e "s~@@@DOWNLOAD_URL@@@~${download_url}~g" \
               < "${local_template}" > "${local_filename}"
            # Create sftp instruction file
            echo "-rm ${remote_filepath}.oldold" > batch
            echo "-rename ${remote_filepath}.old ${remote_filepath}.oldold" >> batch
            echo "-rename ${remote_filepath}.html ${remote_filepath}.old" >> batch
            echo "-put ${local_filename} ${remote_filepath}.html" >> batch
            echo "-put ${local_manpath} ${remote_manpath}" >> batch
            # Do the sftp
            eval $(ssh-agent)
            ssh-add - <<< "${SSH_KEY}"
            sftp -o StrictHostKeyChecking=no -b batch ubuntu@online.interlisp.org
        env:
            SSH_KEY: ${{ secrets.OIO_SSH_KEY }}



# JOB: complete #####################################################################

  # Use set-sentry-action to determine set the sentry that says this release has
  # been successfully built

  complete:

    runs-on: ubuntu-latest

    outputs:
      build_successful: ${{ steps.output.outputs.build_successful }}

    needs: [inputs, sentry, loadup, downloads_page]

    steps:
      # Delete the tarballs artifact
      - name: Delete tarballs artifact
        uses: geekyeggo/delete-artifact@v2
        with:
          name: tarballs
          failOnError: false

      # Checkout the actions for this repo owner
      - name: Checkout Actions
        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/.github
          path: ./Actions_${{ github.sha }}
      - run: mv ./Actions_${{ github.sha }}/actions ../actions && rm -rf ./Actions_${{ github.sha }}

      # Set sentry
      - name: Set flag that build for this commit has been completed
        id: set
        uses: ./../actions/set-sentry-action
        with:
          tag: "loadup"

      - name: Output
        id: output
        run: |
          echo "build_successful='true'" >> $GITHUB_OUTPUT

######################################################################################
######################################################################################