forked from Submitty/Submitty
-
Notifications
You must be signed in to change notification settings - Fork 11
255 lines (216 loc) · 11.3 KB
/
db_check.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
name: Submitty DB Check
on:
pull_request:
branches:
- main
push:
paths:
- migration/**
workflow_dispatch:
jobs:
db-check:
runs-on: ubuntu-20.04
env:
PGPASSWORD: submitty_dbuser
MAIN_INSTALL_PATH: ${{ github.WORKSPACE }}/main
MAIN_DATA_PATH: ${{ github.WORKSPACE }}/main/var
MAIN_REPO_PATH: ${{ github.WORKSPACE }}/main/GIT_CHECKOUT
PR_INSTALL_PATH: ${{ github.WORKSPACE }}/branch
PR_DATA_PATH: ${{ github.WORKSPACE }}/branch/var
PR_REPO_PATH: ${{ github.WORKSPACE }}/branch/GIT_CHECKOUT
services:
postgres-main:
image: postgres:12
env:
POSTGRES_PASSWORD: submitty_dbuser
POSTGRES_USER: postgres
TZ: America/New_York
ports:
- 5432
postgres-branch:
image: postgres:12
env:
POSTGRES_PASSWORD: submitty_dbuser
POSTGRES_USER: postgres
TZ: America/New_York
ports:
- 5432
steps:
- name: Setup Dependencies
run: |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get autoremove -y postgresql-14 postgresql-client-common
sudo apt-get install -y postgresql-12
sudo rm /etc/postgresql/14 -rf
- name: Checkout Current Branch
uses: actions/checkout@v4
with:
path: branch/GIT_CHECKOUT/Submitty
fetch-depth: 0
- name: Find Merge Base
run: |
echo MBASE=$(git --git-dir ${PR_REPO_PATH}/Submitty/.git merge-base \
-a origin/main ${{ github.sha }}) >> $GITHUB_ENV
- name: Checkout main Branch
uses: actions/checkout@v4
with:
ref: ${{ env.MBASE || 'main' }}
path: main/GIT_CHECKOUT/Submitty
- name: Download Required Packages
run: |
pip install -r <(
grep -e "tzlocal" -e "sqlalchemy" -e "ruamel" \
-e "psycopg2" -e "docker" \
${MAIN_REPO_PATH}/Submitty/.setup/pip/system_requirements.txt
)
pip install ${MAIN_REPO_PATH}/Submitty/python_submitty_utils
- name: Create Users and Groups
run: | # Create users and groups
sudo -E python3 ${MAIN_REPO_PATH}/Submitty/.setup/bin/create_untrusted_users.py
sudo addgroup submitty_daemonphp
sudo addgroup submitty_daemoncgi
sudo addgroup submitty_course_builders
sudo addgroup submitty_phpgrp
sudo adduser submitty_php --disabled-password --no-create-home
sudo adduser submitty_cgi --disabled-password --no-create-home
sudo adduser submitty_cgi submitty_phpgrp --no-create-home
sudo adduser submitty_php shadow --no-create-home
sudo adduser submitty_cgi shadow --no-create-home
sudo adduser submitty_daemon --disabled-password --no-create-home
sudo adduser submitty_php submitty_daemonphp --no-create-home
sudo adduser submitty_daemon submitty_daemonphp --no-create-home
sudo adduser submitty_cgi submitty_daemoncgi --no-create-home
sudo adduser submitty_daemon submitty_daemoncgi --no-create-home
sudo adduser submitty_daemon docker --no-create-home
sudo useradd -p $(openssl passwd -1 submitty_dbuser) submitty_dbuser
- name: Configure Submitty
run: |
configure_submitty() {
# $1->MAIN_INSTALL_PATH, $2->MAIN_DATA_PATH, $3->MAIN_REPO_PATH, $4->DB_PORT
mkdir -vp "$2" && chmod -vR 777 "$2"
mkdir -vp "$1/config" && chmod -vR 777 "$1/config"
mkdir -vp "$1/site" && chmod -vR 777 "$1/site"
mkdir -vp "$2/courses" && chmod -vR 777 "$2/courses"
(
echo "localhost" # database host
echo "$4" # database port
echo "" # global db user
echo "submitty_dbuser" # global db pass
echo "" # course db user
echo "submitty_dbuser" # course db pass
echo "" # timezone
echo "" # default locale
echo "http://localhost" # submitty url
echo "" # vcs url
echo "" # institution name
echo "" # sysadmin email
echo "" # where to report
echo "1" # PamAuth
echo "" # sysadmin username
echo "n" # email notification
) | tee /dev/stderr \
| sudo -E python3 "$3/Submitty/.setup/CONFIGURE_SUBMITTY.py" \
--install-dir "$1" --data-dir "$2" --debug
# Copy necessary files to the installation path
cp -vr "$3/Submitty/sbin" "$1"
sudo chmod -vR 777 "$1/config"
sudo chmod -vR 777 "$1/sbin"
}
configure_submitty "${MAIN_INSTALL_PATH}" "${MAIN_DATA_PATH}" \
"${MAIN_REPO_PATH}" "${{ job.services.postgres-main.ports['5432'] }}"
configure_submitty "${PR_INSTALL_PATH}" "${PR_DATA_PATH}" \
"${PR_REPO_PATH}" "${{ job.services.postgres-branch.ports['5432'] }}"
- name: Create Databases
run: |
psql -d postgres -h localhost -U postgres -p ${{ job.services.postgres-main.ports['5432'] }} \
-c "CREATE ROLE submitty_dbuser WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'submitty_dbuser'"\
-c "CREATE ROLE submitty_course_dbuser WITH LOGIN PASSWORD 'submitty_dbuser'"
psql -d postgres -h localhost -U submitty_dbuser -p ${{ job.services.postgres-main.ports['5432'] }} \
-c "CREATE DATABASE submitty"
psql -d postgres -h localhost -U postgres -p ${{ job.services.postgres-branch.ports['5432'] }} \
-c "CREATE ROLE submitty_dbuser WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'submitty_dbuser'"\
-c "CREATE ROLE submitty_course_dbuser WITH LOGIN PASSWORD 'submitty_dbuser'"
psql -d postgres -h localhost -U submitty_dbuser -p ${{ job.services.postgres-branch.ports['5432'] }} \
-c "CREATE DATABASE submitty"
- name: Create Database Dumper
run: | # the original dumper requires a `su`, which is not needed when psql as a service
(
echo 'pg_dump -Osx -d "$1" -f "$2" -h localhost -p "$3" -U postgres '
echo 'sed -i -e "'"/SELECT pg_catalog.set_config('search_path'/d"'"\'
echo ' -e "/-- Dumped \(from\|by\)/d" \'
echo ' -e "/SET default_table_access_method = /d" \'
echo ' -e "s/FOR EACH ROW EXECUTE FUNCTION/FOR EACH ROW EXECUTE PROCEDURE/g" "$2"'
) > dbdump.sh
chmod +x dbdump.sh
echo M=$([ "$(date +%-m)" -lt "7" ] && echo 's' || echo 'f') >> $GITHUB_ENV
echo Y=$(date +%y) >> $GITHUB_ENV
mkdir -vp ${GITHUB_WORKSPACE}/dumped/main
mkdir -vp ${GITHUB_WORKSPACE}/dumped/branch
- name: Setup Databases and Courses for main
run: | # migrate main according to dumped master database (assume it is correct)
python3 ${MAIN_REPO_PATH}/Submitty/migration/run_migrator.py -e master migrate --initial
sudo ln -vs /usr/bin/true /usr/bin/ldapadd
sudo -E python3 ${MAIN_REPO_PATH}/Submitty/.setup/bin/setup_sample_courses.py \
blank --no_grading --data_dir ${MAIN_DATA_PATH} --install_dir ${MAIN_INSTALL_PATH}
sudo chmod -vR 777 ${MAIN_DATA_PATH}/courses
- name: Apply New Migrations from Current Branch
run: |
python3 ${PR_REPO_PATH}/Submitty/migration/run_migrator.py \
-e master -e course -c ${MAIN_INSTALL_PATH}/config migrate
- name: Setup Databases and Courses for Current Branch
run: |
python3 ${PR_REPO_PATH}/Submitty/migration/run_migrator.py -e master migrate --initial
sudo -E python3 ${PR_REPO_PATH}/Submitty/.setup/bin/setup_sample_courses.py \
blank --no_grading --data_dir ${PR_DATA_PATH} --install_dir ${PR_INSTALL_PATH}
sudo chmod -vR 777 ${PR_DATA_PATH}/courses
- name: Dump Migrated Databases
run: |
./dbdump.sh "submitty" "${GITHUB_WORKSPACE}/dumped/main/master.sql"\
"${{ job.services.postgres-main.ports['5432'] }}"
./dbdump.sh "submitty_${M}${Y}_blank" "${GITHUB_WORKSPACE}/dumped/main/course.sql"\
"${{ job.services.postgres-main.ports['5432'] }}"
./dbdump.sh "submitty" "${GITHUB_WORKSPACE}/dumped/branch/master.sql"\
"${{ job.services.postgres-branch.ports['5432'] }}"
./dbdump.sh "submitty_${M}${Y}_blank" "${GITHUB_WORKSPACE}/dumped/branch/course.sql"\
"${{ job.services.postgres-branch.ports['5432'] }}"
- name: Compare Migrated master Databases
if: always()
run: |
diff --color ${GITHUB_WORKSPACE}/dumped/main/master.sql \
${GITHUB_WORKSPACE}/dumped/branch/master.sql || {
echo "::error:: Master databases are not identical, please check your migration"
exit 1
}
- name: Compare Migrated course Databases
if: always()
run: |
diff --color ${GITHUB_WORKSPACE}/dumped/main/course.sql \
${GITHUB_WORKSPACE}/dumped/branch/course.sql || {
echo "::error:: Course databases are not identical, please check your migration"
exit 1
}
- name: Compare Migrated master Database with Provided Dump
if: always()
run: |
diff --color ${GITHUB_WORKSPACE}/dumped/branch/master.sql \
${PR_REPO_PATH}/Submitty/migration/migrator/data/submitty_db.sql || {
echo "::error:: Wrong provided dump, please check your database dumps"
exit 1
}
- name: Compare Migrated course Database with Provided Dump
if: always()
run: |
diff --color ${GITHUB_WORKSPACE}/dumped/branch/course.sql \
${PR_REPO_PATH}/Submitty/migration/migrator/data/course_tables.sql || {
echo "::error:: Wrong provided dump, please check your database dumps"
exit 1
}
- name: Pack Artifacts
if: failure()
uses: actions/upload-artifact@v4
with:
name: dumped_database
path: ${{ github.WORKSPACE }}/dumped/**/*.sql
retention-days: 3