Skip to content

Deploy Delay Notifications #25585

Deploy Delay Notifications

Deploy Delay Notifications #25585

name: Deploy Delay Notifications
on:
schedule:
- cron: "*/10 * * * *" # Runs every 10 minutes
jobs:
check-deployment:
runs-on: ubuntu-latest
outputs:
dev_summary: ${{ steps.check-dev-status.outputs.dev_summary }}
staging_summary: ${{ steps.check-staging-status.outputs.staging_summary }}
steps:
- name: Get latest commit SHA and time from master branch
id: git-info
run: |
latest_commit_info=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
"https://api.github.com/repos/${{ github.repository }}/commits/master")
latest_sha=$(echo "${latest_commit_info}" | jq -r '.sha')
echo "latest_sha=${latest_sha}" >> $GITHUB_ENV
echo "latest_sha: ${latest_sha}"
commit_time=$(echo "${latest_commit_info}" | jq -r '.commit.committer.date')
echo "commit_time=${commit_time}" >> $GITHUB_ENV
echo "commit_time: ${commit_time}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Get deployed SHA for development
id: dev-deploy-sha
run: |
deployed_sha=$(curl -s https://dev-api.va.gov/v0/status | jq -r .git_revision)
echo "dev_deployed_sha=${deployed_sha}" >> $GITHUB_ENV
- name: Get deployed SHA for staging
id: staging-deploy-sha
run: |
deployed_sha=$(curl -s https://staging-api.va.gov/v0/status | jq -r .git_revision)
echo "staging_deployed_sha=${deployed_sha}" >> $GITHUB_ENV
- name: Check deployment status for development
if: ${{ env.latest_sha != '' && env.dev_deployed_sha != '' }}
id: check-dev-status
run: |
latest_sha=${{ env.latest_sha }}
commit_time=${{ env.commit_time }}
deployed_sha=${{ env.dev_deployed_sha }}
info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to development"
action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-dev|ArgoCD dev>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>"
if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then
echo "${info_message} has been deployed."
echo "dev_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '45 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 45 minutes. Skipping notification."
echo "dev_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '30 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 30 minutes."
echo "Current commit on development is ${deployed_sha:0:8}."
echo "dev_summary=${info_message} has been delayed for more than 30 minutes. Current commit on development is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT
exit 1
else
echo "Awaiting deployment of ${info_message}."
echo "dev_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT
fi
- name: Check deployment status for staging
if: ${{ always() && env.latest_sha != '' && env.staging_deployed_sha != '' }}
id: check-staging-status
run: |
latest_sha=${{ env.latest_sha }}
commit_time=${{ env.commit_time }}
deployed_sha=${{ env.staging_deployed_sha }}
info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to staging"
action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-staging|ArgoCD staging>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>"
if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then
echo "${info_message} has been deployed."
echo "staging_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '45 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 45 minutes. Skipping notification."
echo "staging_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '30 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 30 minutes."
echo "Current commit on staging is ${deployed_sha:0:8}."
echo "staging_summary=${info_message} has been delayed for more than 30 minutes. Current commit on staging is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT
exit 1
else
echo "Awaiting deployment of ${info_message}."
echo "staging_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT
fi
notify-on-failure:
runs-on: ubuntu-latest
needs: [check-deployment]
env:
dev_summary: ${{ needs.check-deployment.outputs.dev_summary }}
staging_summary: ${{ needs.check-deployment.outputs.staging_summary }}
if: ${{ failure() }}
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: "us-gov-west-1"
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN
env_variable_name: VA_VSP_BOT_GITHUB_TOKEN
- name: Checkout VSP actions
uses: actions/checkout@v4
with:
repository: department-of-veterans-affairs/vsp-github-actions
ref: refs/heads/main
token: ${{ env.VA_VSP_BOT_GITHUB_TOKEN }}
persist-credentials: false
path: ./.github/actions/vsp-github-actions
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/github_actions_slack_socket_token
env_variable_name: SLACK_APP_TOKEN
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/github_actions_slack_bot_user_token
env_variable_name: SLACK_BOT_TOKEN
- name: Notify for deployment failure
if: ${{ env.dev_summary != '' || env.staging_summary != '' }}
uses: ./.github/actions/vsp-github-actions/slack-socket
with:
slack_app_token: ${{ env.SLACK_APP_TOKEN }}
slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
message: "Vets API Deployment Delay:"
blocks: |
[
{ "type": "divider" },
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>\n\n *Development Summary:*\n${{ env.dev_summary }}\n\n *Staging Summary:*\n${{ env.staging_summary }}"
}
},
{ "type": "divider" }
]
channel_id: "C039HRTHXDH"
- name: Notify for other failure
if: ${{ env.dev_summary == '' && env.staging_summary == '' }}
uses: ./.github/actions/vsp-github-actions/slack-socket
with:
slack_app_token: ${{ env.SLACK_APP_TOKEN }}
slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
message: "Vets API Deployment Delay:"
blocks: |
[
{ "type": "divider" },
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n\n Unknown error occured. See logs:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>"
}
},
{ "type": "divider" }
]
channel_id: "C039HRTHXDH"