tests/screentest: update goldens to latest chromedp version

Use a more recent version of chromedp and screentest. The newer screentest takes arguments differently, so update its invocations. The newer chromedp displays things slightly differently, so update the goldens. Also use the suffix "want" instead of "a", as the new screentest requires. Comment out buggy or flaky tests. These will be handled in followup CLs. Also, improve and enhance run.sh, notably adding the -run flag to run a subset of the tests. Change-Id: Ia46bf02f8cae15a763c80c6d6744f4ac9c005b9b Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/631780 kokoro-CI: kokoro <[email protected]> Reviewed-by: Robert Findley <[email protected]> LUCI-TryBot-Result: Go LUCI <[email protected]>
golang · Nov 26, 2024 · a9be465 · a9be465
1 parent aa5dca7
commit a9be465
Show file tree

Hide file tree

Showing 163 changed files with 109 additions and 85 deletions.
diff --git a/devtools/docker/compose.yaml b/devtools/docker/compose.yaml
@@ -130,7 +130,7 @@ services:
     working_dir: /pkgsite
   chromedp:
     # This should match the version we are using in tests/screentest/run.sh.
-    image: chromedp/headless-shell:97.0.4692.71
+    image: chromedp/headless-shell:131.0.6778.33
     shm_size: 8G
   go:
      # This should match the version we are using on Cloud Run.

diff --git a/tests/README.md b/tests/README.md
@@ -73,8 +73,7 @@ go run tests/api/main.go compare [module path]:[package path suffix]
 
 The screentest/ directory contains visual diff tests for pages on pkg.go.dev.
 Tests are written for the screentest tool from golang.org/x/website.
-See the [command](https://pkg.go.dev/golang.org/x/website/cmd/screentest) and
-[package](https://pkg.go.dev/golang.org/x/website/internal/screentest) docs for
+See the [command](https://pkg.go.dev/golang.org/x/website/cmd/screentest) docs for
 more information.
 
 ### Running the tests
@@ -89,7 +88,7 @@ In order to run the tests, run this command from the root of the repository:
 postgres database, frontend, and headless chrome, and runs the screentest using
 headless chrome.
 
-Tests in `./tests/screentest/testcases.ci.txt` run only for CI runs of the
+Tests in `./tests/screentest/testcases.ci.txt` run only for CI and local runs of the
 screentest. These tests may rely on mocked data, like the vuln page tests.
 Tests in `./tests/screentest/testcases.txt` run for all environments and
 are suitable for tests with data that is consistent between environments or

diff --git a/tests/screentest/run.sh b/tests/screentest/run.sh
@@ -8,10 +8,13 @@ set -e
 
 source devtools/lib.sh || { echo "Are you at repo root?"; exit 1; }
 
-screentest_version=v0.0.0-20241108174919-3a761022ad6f
+# Do not truncate commands when displaying them (see runcmd in devtools/lib.sh).
+MAXWIDTH=0
+
+screentest_version=latest
 
 # This should match the version we are using in devtools/docker/compose.yaml.
-chromedp_version=97.0.4692.71
+chromedp_version=131.0.6778.33
 
 chromedp_port=9222
 frontend_port=8080
@@ -53,6 +56,9 @@ Options:
     You can provide this with no command-line argument to remove resources from a previous
     command that did not specify -rm.
 
+  -run <REGEXP>
+    Run only tests whose names match REGEXP.
+
   -update
     Recapture every snapshot during this test run.
 
@@ -90,9 +96,7 @@ cleanup() {
 
 main() {
   trap cleanup EXIT
-  local concurrency
-  local idtoken
-  local update
+  local concurrency idtoken update run
   while [[ $1 = -* ]]; do
     case "$1" in
       -concurrency)
@@ -104,11 +108,15 @@ main() {
         idtoken=$1
         ;;
      -update)
-        update="-u"
+        update="-update"
         ;;
      -rm)
         rm=true
         ;;
+     -run)
+        shift
+        run="-run '$1'"
+        ;;
       *)
         usage
         exit 1
@@ -123,31 +131,36 @@ main() {
 
   env=$1
   local debugger_url="-d ws://localhost:$chromedp_port"
-  local vars
+  local test_server headers
   case $env in
     ci)
+      test_server="http://frontend:$frontend_port"
       debugger_url="-d ws://chromedp:$chromedp_port"
-      vars="-v Origin:http://frontend:$frontend_port"
       ;;
     exp|dev|staging)
+      test_server="https://$env-pkg.go.dev"
       debugger_url="-d ws://chromedp:$chromedp_port"
-      vars="-v Origin:https://$env-pkg.go.dev,QuotaBypass:$GO_DISCOVERY_E2E_QUOTA_BYPASS,Token:$idtoken"
+      headers="-headers QuotaBypass:$GO_DISCOVERY_E2E_QUOTA_BYPASS,Token:$idtoken"
       ;;
     prod)
-      vars="-v Origin:https://pkg.go.dev,QuotaBypass:$bypass"
+      test_server="https://pkg.go.dev"
+      headers="-headers QuotaBypass:$GO_DISCOVERY_E2E_QUOTA_BYPASS"
+      ;;
+    local)
+      test_server="http://localhost:$frontend_port"
       ;;
-    local) ;;
     *)
       usage
       ;;
   esac
 
-  local testfile="tests/screentest/testcases.txt"
-  local cmd="screentest $concurrency $debugger_url $vars $update $testfile"
+  local testfiles=tests/screentest/testcases.txt
+  if [[ "$env" == ci || "$env" == local ]]; then
+    testfiles=tests/screentest/testcases.*
+  fi
+  local cmd="screentest $concurrency $debugger_url $headers $update $run $test_server tests/screentest/testdata $testfiles"
 
   if [[ "$env" = ci ]]; then
-    testfile="'tests/screentest/testcases.*'"
-    cmd="screentest $concurrency $debugger_url $vars $update $testfile"
     export GO_DISCOVERY_CONFIG_DYNAMIC="tests/screentest/config.yaml"
     export GO_DISCOVERY_DATABASE_NAME="discovery_e2e_test"
     export GO_DISCOVERY_SEED_DB_FILE="tests/screentest/seed.txt"
@@ -160,7 +173,7 @@ main() {
       go run ./devtools/cmd/wait_available --timeout 120s frontend:$frontend_port -- \
       $(echo $cmd)"
   elif [[ "$env" == local ]]; then
-    run_locally $concurrency $update
+    run_locally "$cmd"
   else
     dcompose up --detach chromedp
     dcompose run --rm --entrypoint bash go -c "
@@ -173,11 +186,12 @@ main() {
 # run_locally: run outside of the docker compose network, but use
 # docker for each component.
 run_locally() {
-  local concurrency=$1
-  local update=$2
+  local cmd="$1"
 
   if ! command -v screentest &> /dev/null; then
-    runcmd go install golang.org/x/website/cmd/screentest@screentest_version
+    runcmd go install golang.org/x/website/cmd/screentest@$screentest_version
+  else
+    info using locally installed screentest
   fi
 
   export GO_DISCOVERY_DATABASE_NAME=discovery-db
@@ -218,15 +232,17 @@ run_locally() {
   if ! listening $chromedp_port; then
     info starting chromedp
     runcmd docker run --detach --rm --network host --shm-size 8G \
-          --name headless-shell chromedp/headless-shell:$chromedp_version
+          --name headless-shell \
+          chromedp/headless-shell:$chromedp_version --remote-debugging-port $chromedp_port
     wait_for $chromedp_port
+    # Give some extra time after listening.
+    runcmd sleep 2
+  elif pgrep -u $USER headless-shell > /dev/null; then
+    err "running headless-shell locally will give different results"
   fi
 
   info "running screentest"
-  screentest $concurrency $update \
-    -v Origin:http://localhost:$frontend_port \
-    -d ws://localhost:$chromedp_port \
-    'tests/screentest/testcases.*'
+  runcmd eval $cmd
 }
 
 listening() {
@@ -237,4 +253,4 @@ wait_for() {
 	timeout 5s bash -c -- "while ! nc -z localhost $1; do sleep 1; done"
 }
 
-main $@
+main "$@"
diff --git a/tests/screentest/testcases.ci.txt b/tests/screentest/testcases.ci.txt
@@ -1,6 +1,4 @@
 windowsize 1536x960
-compare {{.Origin}}::cache {{.Origin}}
-output tests/screentest/testdata/ci
 # Prevent dynamic readme bages from rendering
 block https://codecov.io/* https://travis-ci.com/*
 
@@ -9,66 +7,66 @@ block https://codecov.io/* https://travis-ci.com/*
 {{$scrollTop := "window.scrollTo({top:0});"}}
 
 test vuln
-pathname /vuln/
+path /vuln/
 capture fullscreen
 capture fullscreen 540x1080
 
 test vuln list
-pathname /vuln/list
+path /vuln/list
 capture fullscreen
 capture fullscreen 540x1080
-
+ 
 test vuln entry
-pathname /vuln/GO-2022-0229
+path /vuln/GO-2022-0229
 capture fullscreen
 capture fullscreen 540x1080
 
 test vuln entry withdrawn
-pathname /vuln/GO-2024-2730
+path /vuln/GO-2024-2730
 capture fullscreen
 capture fullscreen 540x1080
 
 test vuln entry no packages
-pathname /vuln/GO-2024-2659
+path /vuln/GO-2024-2659
 capture fullscreen
 capture fullscreen 540x1080
 
 test vuln entry custom ranges module
-pathname /vuln/GO-2024-2979
+path /vuln/GO-2024-2979
 capture fullscreen
 capture fullscreen 540x1080
 
 test vuln entry custom ranges package
-pathname /vuln/GO-2024-2527
+path /vuln/GO-2024-2527
 capture fullscreen
 capture fullscreen 540x1080
 
 test vuln entry unreviewed
-pathname /vuln/GO-2024-2864
+path /vuln/GO-2024-2864
 capture fullscreen
 capture fullscreen 540x1080
 
 test vuln stdlib module
-pathname /archive/[email protected]
+path /archive/[email protected]
 capture viewport
 capture viewport 540x1080
 
 test vuln toolchain module
-pathname /cmd/[email protected]
+path /cmd/[email protected]
 capture viewport
 capture viewport 540x1080
 
 test vuln nonstdlib module
-pathname /golang.org/x/crypto?tab=versions
+path /golang.org/x/crypto?tab=versions
 capture viewport
 capture viewport 540x1080
 
 test vuln search
-pathname /search?q=github.com%2Fbeego&m=vuln
+path /search?q=github.com%2Fbeego&m=vuln
 capture viewport
 capture viewport 540x1080
 
 test vuln no results
-pathname /search?q=github.com%2Fnoresults&m=vuln
+path /search?q=github.com%2Fnoresults&m=vuln
 capture viewport
 capture viewport 540x1080