-
Notifications
You must be signed in to change notification settings - Fork 5
142 lines (118 loc) · 6.23 KB
/
s3-backup-retention.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
name: VAgov CMS S3 Backup Manager
on:
# UTC 5am is ET 1am, everyday.
schedule:
- cron: '0 5 * * 1-5'
workflow_dispatch:
jobs:
backup-daily:
if: github.repository == 'department-of-veterans-affairs/va.gov-cms'
runs-on: ubuntu-latest
steps:
# Cron set to run daily. Lets get the day of the week for the weekly backup steps.
- name: Get current date
id: date
run: echo "date=$(date)" >> $GITHUB_OUTPUT
- name: Display date
run: echo "The current date is ${{ steps.date.outputs.date }}"
# Get the initial AWS IAM User credentials. Only has basic permissions for sts:assumeRole
- name: Configure AWS credentials (1)
uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-gov-west-1
# Will use in the future. CMS github actions user needs additional permissions to use SSM Parameter Store.
# - name: Get AWS IAM role
# uses: department-of-veterans-affairs/action-inject-ssm-secrets@latest
# with:
# ssm_parameter: /cms/github-actions/parameters/AWS_VAGOV_CMS_PROD_S3_ROLE
# env_variable_name: AWS_VAGOV_CMS_PROD_S3_ROLE
# Get credentials from our s3 role. Least privilege method for AWS IAM.
- name: Configure AWS credentials (1)
uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-gov-west-1
role-to-assume: ${{ secrets.AWS_VAGOV_CMS_PROD_S3_ROLE }}
role-duration-seconds: 900
role-session-name: vsp-vagov-cms-githubaction
# Daily Backups
- name: Backup Daily Asset Files
run: |
latest_file=$(aws s3api list-objects \
--bucket $BUCKET \
--prefix $SOURCE_PREFIX \
--query "Contents[?contains(Key, 'cmsapp')] | reverse(sort_by(@, &LastModified)[].{LastModified:LastModified,Key:Key}) | [:1]" | jq '.[].Key' --raw-output)
latest_file_no_prefix=$(echo $latest_file | sed "s%^$SOURCE_PREFIX%%g")
aws s3 cp s3://$BUCKET/$latest_file s3://$BUCKET/$DESTINATION_PREFIX/$latest_file_no_prefix
env:
SOURCE_PREFIX: files/
DESTINATION_PREFIX: backups/daily/files
BUCKET: dsva-vagov-prod-cms-backup-sanitized
- name: Backup Daily Sanitized SQL Files
run: |
latest_file=$(aws s3api list-objects \
--bucket $BUCKET \
--prefix $SOURCE_PREFIX \
--query "Contents[?contains(Key, 'latest') == \`false\`] | reverse(sort_by(@, &LastModified)[].{LastModified:LastModified,Key:Key}) | [:1]" | jq '.[].Key' --raw-output)
latest_file_no_prefix=$(echo $latest_file | sed "s%^$SOURCE_PREFIX%%g")
aws s3 cp s3://$BUCKET/$latest_file s3://$BUCKET/$DESTINATION_PREFIX/$latest_file_no_prefix
env:
SOURCE_PREFIX: database/
DESTINATION_PREFIX: backups/daily/database
BUCKET: dsva-vagov-prod-cms-backup-sanitized
- name: Backup Daily Production SQL Files
run: |
latest_file=$(aws s3api list-objects \
--bucket $BUCKET \
--prefix $SOURCE_PREFIX \
--query "Contents[?contains(Key, 'latest') == \`false\`] | reverse(sort_by(@, &LastModified)[].{LastModified:LastModified,Key:Key}) | [:1]" | jq '.[].Key' --raw-output)
latest_file_no_prefix=$(echo $latest_file | sed "s%^$SOURCE_PREFIX%%g")
aws s3 cp s3://$BUCKET/$latest_file s3://$BUCKET/$DESTINATION_PREFIX/$latest_file_no_prefix
env:
SOURCE_PREFIX: database/
DESTINATION_PREFIX: backups/daily/database
BUCKET: dsva-vagov-prod-cms-backup
# Weekly
# if: contains(steps.date.outputs.date,'Mon'), if the date returned is Monday, run these steps.
- name: Backup Weekly Asset Files
if: contains(steps.date.outputs.date,'Mon')
run: |
latest_file=$(aws s3api list-objects \
--bucket $BUCKET \
--prefix $SOURCE_PREFIX \
--query "Contents[?contains(Key, 'cmsapp')] | reverse(sort_by(@, &LastModified)[].{LastModified:LastModified,Key:Key}) | [:1]" | jq '.[].Key' --raw-output)
latest_file_no_prefix=$(echo $latest_file | sed "s%^$SOURCE_PREFIX%%g")
aws s3 cp s3://$BUCKET/$latest_file s3://$BUCKET/$DESTINATION_PREFIX/$latest_file_no_prefix
env:
SOURCE_PREFIX: files/
DESTINATION_PREFIX: backups/weekly/files
BUCKET: dsva-vagov-prod-cms-backup-sanitized
- name: Backup Weekly Sanitized SQL Files
if: contains(steps.date.outputs.date,'Mon')
run: |
latest_file=$(aws s3api list-objects \
--bucket $BUCKET \
--prefix $SOURCE_PREFIX \
--query "Contents[?contains(Key, 'latest') == \`false\`] | reverse(sort_by(@, &LastModified)[].{LastModified:LastModified,Key:Key}) | [:1]" | jq '.[].Key' --raw-output)
latest_file_no_prefix=$(echo $latest_file | sed "s%^$SOURCE_PREFIX%%g")
aws s3 cp s3://$BUCKET/$latest_file s3://$BUCKET/$DESTINATION_PREFIX/$latest_file_no_prefix
env:
SOURCE_PREFIX: database/
DESTINATION_PREFIX: backups/weekly/database
BUCKET: dsva-vagov-prod-cms-backup-sanitized
- name: Backup Weekly Production SQL Files
if: contains(steps.date.outputs.date,'Mon')
run: |
latest_file=$(aws s3api list-objects \
--bucket $BUCKET \
--prefix $SOURCE_PREFIX \
--query "Contents[?contains(Key, 'latest') == \`false\`] | reverse(sort_by(@, &LastModified)[].{LastModified:LastModified,Key:Key}) | [:1]" | jq '.[].Key' --raw-output)
latest_file_no_prefix=$(echo $latest_file | sed "s%^$SOURCE_PREFIX%%g")
aws s3 cp s3://$BUCKET/$latest_file s3://$BUCKET/$DESTINATION_PREFIX/$latest_file_no_prefix
env:
SOURCE_PREFIX: database/
DESTINATION_PREFIX: backups/weekly/database
BUCKET: dsva-vagov-prod-cms-backup