-
Notifications
You must be signed in to change notification settings - Fork 66
135 lines (118 loc) · 7.48 KB
/
deploy_delay_notifications.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
name: Deploy Delay Notifications
on:
schedule:
- cron: "*/10 * * * *" # Runs every 10 minutes
jobs:
check-deployment:
runs-on: ubuntu-latest
outputs:
development_summary: ${{ steps.check-dev-status.outputs.dev_summary }}
staging_summary: ${{ steps.check-staging-status.outputs.staging_summary }}
steps:
- name: Get latest commit SHA and time from master branch
id: git-info
run: |
latest_commit_info=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
"https://api.github.com/repos/${{ github.repository }}/commits/master")
latest_sha=$(echo "${latest_commit_info}" | jq -r '.sha')
commit_time=$(echo "${latest_commit_info}" | jq -r '.commit.committer.date')
echo "latest_sha=${latest_sha}" >> $GITHUB_ENV
echo "commit_time=${commit_time}" >> $GITHUB_ENV
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Get deployed SHA for development
id: dev-deploy-sha
run: |
deployed_sha=$(curl -s https://dev-api.va.gov/v0/status | jq -r .git_revision)
echo "dev_deployed_sha=${deployed_sha}" >> $GITHUB_ENV
- name: Get deployed SHA for staging
id: staging-deploy-sha
run: |
deployed_sha=$(curl -s https://staging-api.va.gov/v0/status | jq -r .git_revision)
echo "staging_deployed_sha=${deployed_sha}" >> $GITHUB_ENV
- name: Check deployment status for development
id: check-dev-status
run: |
latest_sha=${{ env.latest_sha }}
commit_time=${{ env.commit_time }}
deployed_sha=${{ env.dev_deployed_sha }}
info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to development"
action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-dev|ArgoCD dev>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>"
if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then
echo "${info_message} has been deployed."
echo "dev_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -gt "$(date -d '45 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 45 minutes. Skipping notification."
echo "dev_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -gt "$(date -d '30 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 30 minutes."
echo "Current commit on development is ${deployed_sha:0:8}."
echo "dev_summary=${info_message} has been delayed for more than 30 minutes. Current commit on development is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT
exit 1
else
echo "Awaiting deployment of ${info_message}."
echo "dev_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT
fi
- name: Check deployment status for staging
if: always()
id: check-staging-status
run: |
latest_sha=${{ env.latest_sha }}
commit_time=${{ env.commit_time }}
deployed_sha=${{ env.staging_deployed_sha }}
info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to staging"
action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-staging|ArgoCD staging>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>"
if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then
echo "${info_message} has been deployed."
echo "staging_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -gt "$(date -d '45 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 45 minutes. Skipping notification."
echo "staging_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -gt "$(date -d '30 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 30 minutes."
echo "Current commit on staging is ${deployed_sha:0:8}."
echo "staging_summary=${info_message} has been delayed for more than 30 minutes. Current commit on staging is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT
exit 1
else
echo "Awaiting deployment of ${info_message}."
echo "staging_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT
fi
notify-on-failure:
runs-on: ubuntu-latest
needs: [check-deployment]
if: ${{ failure() }}
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: "us-gov-west-1"
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN
env_variable_name: VA_VSP_BOT_GITHUB_TOKEN
- name: Checkout VSP actions
uses: actions/checkout@v4
with:
repository: department-of-veterans-affairs/vsp-github-actions
ref: refs/heads/main
token: ${{ env.VA_VSP_BOT_GITHUB_TOKEN }}
persist-credentials: false
path: ./.github/actions/vsp-github-actions
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/github_actions_slack_socket_token
env_variable_name: SLACK_APP_TOKEN
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/github_actions_slack_bot_user_token
env_variable_name: SLACK_BOT_TOKEN
- name: Slack notify
uses: ./.github/actions/vsp-github-actions/slack-socket
with:
slack_app_token: ${{ env.SLACK_APP_TOKEN }}
slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
message: "Vets API Deployment Delay:"
blocks: "[{\"type\": \"divider\"}, {\"type\": \"section\", \"text\": { \"type\": \"mrkdwn\", \"text\": \":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>\n\n*Development Summary:*\n${{ needs.check-deployment.outputs.development_summary }}\n\n*Staging Summary:*\n${{ needs.check-deployment.outputs.staging_summary }}\"}}, {\"type\": \"divider\"}]"
channel_id: "C039HRTHXDH"