forked from mathquill/mathquill
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathcircle.yml
188 lines (170 loc) · 7.98 KB
/
circle.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
# Okay so maybe everyone else already knows all this, but it took some time
# for Michael and I [Han] to really see how everything fits together.
#
# Basically, what we're doing here is automated browser testing, so CircleCI
# handles the automation, and Sauce Labs handles the browser testing.
# Specifically, Sauce Labs offers a REST API to run tests in browsers in VMs,
# and CircleCI can be configured to listen for git pushes and run local
# servers and call out to REST APIs to test against these local servers.
#
# The flow goes like this:
# - CircleCI notices/is notified of a git push
# - they pull and checkout and magically know to install dependencies and shit
# + https://circleci.com/docs/manually/
# - their magic works fine for MathQuill's dependencies but to run the tests,
# it foolishly runs `make test`, what an inconceivable mistake
# - that's where we come in: `circle.yml` lets us override the test script.
# + https://circleci.com/docs/configuration/
# - our `circle.yml` first installs and runs a tunnel to Sauce Labs
# - and runs `make server`
# - then it calls out to Sauce Labs' REST API to open a browser that reaches
# back through the tunnel to access the unit test page on the local server
# + > Sauce Connect allows you to run a test server within the CircleCI
# > build container and expose it it (using a URL like `localhost:8080`)
# > to Sauce Labs’ browsers.
#
# https://circleci.com/docs/browser-testing-with-sauce-labs/
#
# - boom testing boom
# this file is based on https://github.com/circleci/sauce-connect/blob/a65e41c91e02550ce56c75740a422bebc4acbf6f/circle.yml
# via https://circleci.com/docs/browser-testing-with-sauce-labs/
dependencies:
cache_directories:
- ~/sauce-connect
pre:
# imagemagick is installed to give us access to the
# `convert` tool to stitch together the screenshots.
- sudo apt-get update; sudo apt-get install imagemagick
- ? |-
mkdir -p ~/sauce-connect
cd ~/sauce-connect
if [ ! -x sc-*-linux/bin/sc ]
then
wget https://saucelabs.com/downloads/sc-latest-linux.tar.gz
tar -xzf sc-latest-linux.tar.gz
fi
sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY --readyfile ~/sauce_is_ready
:
background: true
test:
override:
# Sauce can connect to Safari on ports 3000, 4000, 7000, and 8000. Edge needs port 7000 or 8000.
# https://david263a.wordpress.com/2015/04/18/fixing-safari-cant-connect-to-localhost-issue-when-using-sauce-labs-connect-tunnel/
# https://support.saucelabs.com/customer/portal/questions/14368823-requests-to-localhost-on-microsoft-edge-are-failing-over-sauce-connect
- PORT=8000 make server:
background: true
# CircleCI expects test results to be reported in an JUnit/xUnit-style XML
# file:
# https://circleci.com/docs/test-metadata/
# Our unit tests are in a browser, so they can't write to a file, and Sauce
# apparently truncates custom data in their test result reports, so instead
# we POST to this trivial Node server on localhost:9000 that writes the
# body of any POST request to $CIRCLE_TEST_REPORTS/mocha/xunit.xml
- ? |-
mkdir -p $CIRCLE_TEST_REPORTS/mocha
node << 'EOF' | tee $CIRCLE_TEST_REPORTS/mocha/xunit.xml
require('http').createServer(function(req, res) {
res.setHeader('Access-Control-Allow-Origin', '*');
req.pipe(process.stdout);
req.on('end', res.end.bind(res));
})
.listen(9000);
EOF
:
background: true
# Wait for tunnel to be ready (`make server` and the trivial Node server
# are much faster, no need to wait for them)
- while [ ! -e ~/sauce_is_ready ]; do sleep 1; done
# Start taking screenshots in the background while the unit tests are running
- npm install wd && node script/screenshots.js http://localhost:8000/test/visual.html && touch ~/screenshots_are_ready:
background: true
# Run in-browser unit tests, based on:
# https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods
# "build" and "tag" parameters from:
# https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-TestAnnotation
- |-
curl -i https://saucelabs.com/rest/v1/mathquill/js-tests \
-X POST \
-u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \
-H 'Content-Type: application/json' \
-d '{
"build": "'$(git rev-parse HEAD)'",
"tags": [
"after-v'$(node -p 'require("./package.json").version')'",
"circle-ci"
],
"framework": "mocha",
"url": "http://localhost:8000/test/unit.html?post_xunit_to=http://localhost:9000",
"platforms": [["", "Chrome", ""]]
}' | tee js-tests
# Wait for tests to finish:
#
# > Make the request multiple times as the tests run until the response
# > contains `completed: true` to the get the final results.
#
# https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods
- |-
while true # Bash has no do...while >:(
do
sleep 5
curl -i https://saucelabs.com/rest/v1/mathquill/js-tests/status \
-X POST \
-u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \
-H 'Content-Type: application/json' \
-d "$(tail -1 js-tests)" \
| tee status
tail -1 status > status.json
# deliberately do `... != false` rather than `... == true`
# because unexpected values should break rather than infinite loop
[ "$(node -p 'require("./status.json").completed')" != false ] && break
done
# Wait for screenshots to be ready
- while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done:
timeout: 300
# Stitch together images
# TODO: Split this into multiple yaml lines. I (Michael)
# niavely tried to split this into mutiple yaml lines
# but was unsucessful in doing do.
- |-
img_dir=$CIRCLE_ARTIFACTS/imgs/
for x in $(ls $img_dir)
do
convert $img_dir/$x/*.png -append $img_dir/$x.png
done
# Remove all directories in $CIRCLE_ARTIFACTS/img
# Currently the pieces aren't kept around. If it's
# desirable to keep them around, we should use
# cp -r $dir $CIRCLE_ARTIFACTS/img_pieces
# The reason the image pieces aren't currently kept
# around is that it was leading to a problem. Specifically,
# when we get the previous images, we niavely grab any *.png,
# including the pieces images. This compounded so that each
# iteration of a test run would have all of the images from
# the previous test run plus whichever new images were generated.
rm -R -- $img_dir/*/
# Install utility we need
npm install -g json
# Download the latest mathquill artifacts.
curl $(curl https://circleci.com/api/v1/project/mathquill/mathquill/latest/artifacts \
| json -a url pretty_path -d '\n\t' \
| grep '\.png$' \
| grep -v 'PREV' \
| sed "s:\$CIRCLE_ARTIFACTS/imgs/:-o $img_dir/PREV_:")
# Generate image diffs.
cd $img_dir
for file in $(ls PINNED*); do
prev=PREV_$file
metric_diff=$(compare -metric AE -compose src $prev $file raw_diff.png)
composite -alpha on raw_diff.png $prev DIFF_$file
done
for file in $(ls EVERGREEN*); do
prev=$(ls PREV_$file*)
metric_diff=$(compare -metric AE -compose src $prev $file raw_diff.png)
composite -alpha on raw_diff.png $prev DIFF_$file
done
rm raw_diff.png
# finally, complain to Circle CI if there were nonzero test failures
- |-
[ "$(node -p 'require("./status.json")["js tests"][0].result.failures')" == 0 ]
post:
- killall --wait sc; true # wait for Sauce Connect to close the tunnel; ignore errors since it's just cleanup