From 1ceb32cab097ed5b51382eb1faa61a14800e53be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Audun-Marius=20Gangst=C3=B8?= Date: Fri, 5 Apr 2024 10:50:09 +0900 Subject: [PATCH 1/7] Varnish 5 is quite old and most distros carry 6.0 or newer. --- docs/varnish/README.md | 6 +++--- docs/varnish/vcl/{varnish5.vcl => varnish6.vcl} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename docs/varnish/vcl/{varnish5.vcl => varnish6.vcl} (100%) diff --git a/docs/varnish/README.md b/docs/varnish/README.md index af35befa..ac4e5d1f 100644 --- a/docs/varnish/README.md +++ b/docs/varnish/README.md @@ -3,15 +3,15 @@ eZ Platform Varnish configuration Prerequisites ------------- -* A working Varnish 5.1 and higher _(6.0LTS recommended)_ +* A working Varnish 6.0 or higher _(6.0LTS recommended)_ * With 'xkey' VMOD, correct version is provided with `varnish-modules` 0.10.2 or higher -* Varnish Plus comes with xkey out of the box and can also be used. +* Varnish Enterprise comes with xkey out of the box and can also be used. Recommended VCL base files -------------------------- For Varnish to work properly with eZ, you'll need to use the provided configuration: -* [eZ Platform 2.5LTs optimized Varnish 5.1+ VCL](vcl/varnish5.vcl) +* [eZ Platform 2.5LTs optimized Varnish 6.0+ VCL](vcl/varnish6.vcl) For tuning the VCL further to you needs, see the following relevant examples: - [FOSHttpCache documentation](https://foshttpcache.readthedocs.io/en/latest/varnish-configuration.html) diff --git a/docs/varnish/vcl/varnish5.vcl b/docs/varnish/vcl/varnish6.vcl similarity index 100% rename from docs/varnish/vcl/varnish5.vcl rename to docs/varnish/vcl/varnish6.vcl From cbfb37d47eda97984f2b618adb3cc117079b0f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Audun-Marius=20Gangst=C3=B8?= Date: Mon, 8 Apr 2024 17:52:02 +0900 Subject: [PATCH 2/7] Better grace handling for varnish 6: This code is different with Varnish 6, since if ttl+grace is expired, vcl_hit will not be called. --- docs/varnish/vcl/varnish6.vcl | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/docs/varnish/vcl/varnish6.vcl b/docs/varnish/vcl/varnish6.vcl index 8b0edd6d..2745ab87 100644 --- a/docs/varnish/vcl/varnish6.vcl +++ b/docs/varnish/vcl/varnish6.vcl @@ -71,36 +71,14 @@ sub vcl_recv { // Retrieve client user context hash and add it to the forwarded request. call ez_user_context_hash; + if (std.healthy(req.backend_hint)) { + // change the behavior for healthy backends: Cap grace to 10s + set req.grace = 10s; + } // If it passes all these tests, do a lookup anyway. return (hash); } -// Called when a cache lookup is successful. The object being hit may be stale: It can have a zero or negative ttl with only grace or keep time left. -sub vcl_hit { - if (obj.ttl >= 0s) { - // A pure unadulterated hit, deliver it - return (deliver); - } - - if (obj.ttl + obj.grace > 0s) { - // Object is in grace, logic below in this block is what differs from default: - // https://varnish-cache.org/docs/5.2/users-guide/vcl-grace.html#grace-mode - if (!std.healthy(req.backend_hint)) { - // Service is unhealthy, deliver from cache - return (deliver); - } else if (req.http.cookie) { - // Request it by a user with session, refresh the cache to avoid issues for editors and forum users - return (miss); - } - - // By default deliver cache, automatically triggers a background fetch - return (deliver); - } - - // fetch & deliver once we get the result - return (miss); -} - // Called when the requested object has been retrieved from the backend sub vcl_backend_response { @@ -116,7 +94,7 @@ sub vcl_backend_response { set beresp.do_esi = true; } - // Make Varnish keep all objects for up to 1 hour beyond their TTL, see vcl_hit for Request logic on this + // Make Varnish keep all objects for up to 1 hour beyond their TTL, to serve in case the backend is down set beresp.grace = 1h; // Compressing the content From bb8dc82ea9a2e63918b1d07be70806aff17255c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Audun-Marius=20Gangst=C3=B8?= Date: Mon, 8 Apr 2024 17:59:55 +0900 Subject: [PATCH 3/7] Better handling for the X-Cache-header Adding cases for more than hit/miss, and adding whether varnish deems the content cacheable/uncacheable to aid in debugging cache headers. --- docs/varnish/vcl/varnish6.vcl | 36 ++++++++++++++++++++++++++++++----- docs/varnish/vcl/varnish7.vcl | 36 ++++++++++++++++++++++++++++++----- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/docs/varnish/vcl/varnish6.vcl b/docs/varnish/vcl/varnish6.vcl index 2745ab87..1d67f18e 100644 --- a/docs/varnish/vcl/varnish6.vcl +++ b/docs/varnish/vcl/varnish6.vcl @@ -14,6 +14,8 @@ include "parameters.vcl"; // Called at the beginning of a request, after the complete request has been received sub vcl_recv { + // debug header, unset to make sure it's empty: + unset req.http.x-cache; // Set the backend set req.backend_hint = ezplatform; @@ -79,6 +81,27 @@ sub vcl_recv { return (hash); } +// Set debug header to the appropriate value: +sub vcl_hit { + set req.http.x-cache = "hit"; +} + +sub vcl_miss { + set req.http.x-cache = "miss"; +} + +sub vcl_pass { + set req.http.x-cache = "pass"; +} + +sub vcl_pipe { + set req.http.x-cache = "pipe uncacheable"; +} + +sub vcl_synth { + set resp.http.x-cache = "synth synth"; +} + // Called when the requested object has been retrieved from the backend sub vcl_backend_response { @@ -294,15 +317,18 @@ sub vcl_deliver { } } + if (obj.uncacheable) { + set req.http.x-cache = req.http.x-cache + " uncacheable" ; + } else { + set req.http.x-cache = req.http.x-cache + " cached" ; + } if (client.ip ~ debuggers) { - // Add X-Cache header if debugging is enabled - if (obj.hits > 0) { - set resp.http.X-Cache = "HIT"; + // Add X-Cache (and other) debug header(s) if debugging is enabled + if (req.http.x-cache ~ "^hit") { set resp.http.X-Cache-Hits = obj.hits; set resp.http.X-Cache-TTL = obj.ttl; - } else { - set resp.http.X-Cache = "MISS"; } + set resp.http.x-cache = req.http.x-cache; } else { // Remove tag headers when delivering to non debug client unset resp.http.xkey; diff --git a/docs/varnish/vcl/varnish7.vcl b/docs/varnish/vcl/varnish7.vcl index 3613946b..b3c8d9d1 100644 --- a/docs/varnish/vcl/varnish7.vcl +++ b/docs/varnish/vcl/varnish7.vcl @@ -14,6 +14,8 @@ include "/etc/varnish/parameters.vcl"; // Called at the beginning of a request, after the complete request has been received sub vcl_recv { + // debug header, unset to make sure it's empty: + unset req.http.x-cache; // Set the backend set req.backend_hint = ezplatform; @@ -75,6 +77,27 @@ sub vcl_recv { return (hash); } +// Set debug header to the appropriate value: +sub vcl_hit { + set req.http.x-cache = "hit"; +} + +sub vcl_miss { + set req.http.x-cache = "miss"; +} + +sub vcl_pass { + set req.http.x-cache = "pass"; +} + +sub vcl_pipe { + set req.http.x-cache = "pipe uncacheable"; +} + +sub vcl_synth { + set resp.http.x-cache = "synth synth"; +} + // Called when the requested object has been retrieved from the backend sub vcl_backend_response { @@ -293,15 +316,18 @@ sub vcl_deliver { } } + if (obj.uncacheable) { + set req.http.x-cache = req.http.x-cache + " uncacheable" ; + } else { + set req.http.x-cache = req.http.x-cache + " cached" ; + } if (client.ip ~ debuggers) { - // Add X-Cache header if debugging is enabled - if (obj.hits > 0) { - set resp.http.X-Cache = "HIT"; + // Add X-Cache (and other) debug header(s) if debugging is enabled + if (req.http.x-cache ~ "^hit") { set resp.http.X-Cache-Hits = obj.hits; set resp.http.X-Cache-TTL = obj.ttl; - } else { - set resp.http.X-Cache = "MISS"; } + set resp.http.x-cache = req.http.x-cache; } else { // Remove tag headers when delivering to non debug client unset resp.http.xkey; From e91080af4ead4f0387c78603ec1b7a0063dba390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Audun-Marius=20Gangst=C3=B8?= Date: Mon, 8 Apr 2024 19:02:26 +0900 Subject: [PATCH 4/7] Using vmod cookie to improve readability and performance --- docs/varnish/vcl/varnish7.vcl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/varnish/vcl/varnish7.vcl b/docs/varnish/vcl/varnish7.vcl index b3c8d9d1..248653d5 100644 --- a/docs/varnish/vcl/varnish7.vcl +++ b/docs/varnish/vcl/varnish7.vcl @@ -8,6 +8,7 @@ vcl 4.1; import std; import xkey; +import cookie; // For customizing your backend and acl rules see parameters.vcl include "/etc/varnish/parameters.vcl"; @@ -49,11 +50,9 @@ sub vcl_recv { // Remove all cookies besides Session ID, as JS tracker cookies and so will make the responses effectively un-cached if (req.http.cookie) { - set req.http.cookie = ";" + req.http.cookie; - set req.http.cookie = regsuball(req.http.cookie, "; +", ";"); - set req.http.cookie = regsuball(req.http.cookie, ";(eZSESSID[^=]*)=", "; \1="); - set req.http.cookie = regsuball(req.http.cookie, ";[^ ][^;]*", ""); - set req.http.cookie = regsuball(req.http.cookie, "^[; ]+|[; ]+$", ""); + cookie.parse(req.http.cookie); + cookie.keep("eZSESSID"); + set req.http.cookie = cookie.get_string(); if (req.http.cookie == "") { // If there are no more cookies, remove the header to get page cached. From 7905c329b85deaa70c91b5aea0bd5c97e0930379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Audun-Marius=20Gangst=C3=B8?= Date: Thu, 18 Apr 2024 10:44:43 +0900 Subject: [PATCH 5/7] Varnish 7: vcl_path includes /etc/varnish by default. Not using full path for includes also allows for VCL outside of /etc/varnish, for testing or more flexible configuration. --- docs/varnish/vcl/varnish7.vcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/varnish/vcl/varnish7.vcl b/docs/varnish/vcl/varnish7.vcl index 248653d5..8c46d6bb 100644 --- a/docs/varnish/vcl/varnish7.vcl +++ b/docs/varnish/vcl/varnish7.vcl @@ -11,7 +11,7 @@ import xkey; import cookie; // For customizing your backend and acl rules see parameters.vcl -include "/etc/varnish/parameters.vcl"; +include "parameters.vcl"; // Called at the beginning of a request, after the complete request has been received sub vcl_recv { From 7994dce139057984d32abe67495bb5ca8245dbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Audun-Marius=20Gangst=C3=B8?= Date: Fri, 19 Apr 2024 10:38:17 +0900 Subject: [PATCH 6/7] Add some simple tests of the VCL files --- tests/varnishtests/001-simple-request.vtc | 20 ++++++++++++++++++++ tests/varnishtests/002-cookie-handling.vtc | 22 ++++++++++++++++++++++ tests/varnishtests/README.md | 9 +++++++++ 3 files changed, 51 insertions(+) create mode 100644 tests/varnishtests/001-simple-request.vtc create mode 100644 tests/varnishtests/002-cookie-handling.vtc create mode 100644 tests/varnishtests/README.md diff --git a/tests/varnishtests/001-simple-request.vtc b/tests/varnishtests/001-simple-request.vtc new file mode 100644 index 00000000..7866a5c3 --- /dev/null +++ b/tests/varnishtests/001-simple-request.vtc @@ -0,0 +1,20 @@ +varnishtest "Simple request" + + +server s1 { + rxreq + txresp +} -start + + +shell { + sed "s/.host =.*/.host = \"${s1_addr}\";/;s/.port = .*/.port = \"${s1_port}\";/" ${pwd}/docs/varnish/vcl/parameters.vcl > ${tmpdir}/parameters.vcl + echo "${s1_addr} ${s1_port}" +} -run + +varnish v1 -arg "-p vcl_path=${tmpdir} -f $PWD/docs/varnish/vcl/${vclfile}" -start + +client c1 { + txreq + rxresp +} -run diff --git a/tests/varnishtests/002-cookie-handling.vtc b/tests/varnishtests/002-cookie-handling.vtc new file mode 100644 index 00000000..5aa134ec --- /dev/null +++ b/tests/varnishtests/002-cookie-handling.vtc @@ -0,0 +1,22 @@ +varnishtest "Simple request" + + +server s1 { + rxreq + expect req.http.cookie == "eZSESSID=session" + txresp +} -start + + +shell { + sed "s/.host =.*/.host = \"${s1_addr}\";/;s/.port = .*/.port = \"${s1_port}\";/" ${pwd}/docs/varnish/vcl/parameters.vcl > ${tmpdir}/parameters.vcl + echo "${s1_addr} ${s1_port}" +} -run + +varnish v1 -arg "-p vcl_path=${tmpdir} -f $PWD/docs/varnish/vcl/${vclfile}" -start + +client c1 { + txreq -hdr "Cookie: unrelated_cookie=something; eZSESSID=session; another_cooke=something_else;" + rxresp +} -run + diff --git a/tests/varnishtests/README.md b/tests/varnishtests/README.md new file mode 100644 index 00000000..048bc9a6 --- /dev/null +++ b/tests/varnishtests/README.md @@ -0,0 +1,9 @@ +# Varnishtest files + +Varnishtests can be ran using the official docker images: + +Varnish6: + docker run --rm --entrypoint varnishtest --volume $PWD:/etc/varnish varnish:stable -t 5 -D vclfile=varnish6.vcl tests/varnishtests/001-simple-request.vtc + +Varnish7: + docker run --rm --entrypoint varnishtest --volume $PWD:/etc/varnish varnish:fresh -t 5 -D vclfile=varnish7.vcl tests/varnishtests/001-simple-request.vtc From 064191b2b0e75f2fe90d87ca4edff5030917c022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Audun-Marius=20Gangst=C3=B8?= Date: Wed, 24 Apr 2024 09:32:41 +0900 Subject: [PATCH 7/7] Expect status 200 OK --- tests/varnishtests/001-simple-request.vtc | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/varnishtests/001-simple-request.vtc b/tests/varnishtests/001-simple-request.vtc index 7866a5c3..c277c5e3 100644 --- a/tests/varnishtests/001-simple-request.vtc +++ b/tests/varnishtests/001-simple-request.vtc @@ -17,4 +17,5 @@ varnish v1 -arg "-p vcl_path=${tmpdir} -f $PWD/docs/varnish/vcl/${vclfile}" -sta client c1 { txreq rxresp + expect resp.status == 200 } -run