Varnish doesn't keep serving cache when the backends are down - caching

Question

I have Varnish running behind HAProxy and in front of NGINX. The HAProxy server deals with SSL, then forwards the traffic on port 80 to Varnish, which in turn refers to the NGINX servers.
I'd like to set Varnish up so that if the NGINX servers are all down, it continues to serve the cached content until they're back up. But I don't seem to be able to get it quite right. I'm running varnish-6.0.8 revision 97e54ada6ac578af332e52b44d2038bb4fa4cd4a. My VCL version is 4.1.
Here is my configuration (sanitized):
vcl 4.1;
import directors;
import std;
# Define Server A
backend server-a {
.host = "xx.xx.xx.xx";
.port = "80";
.max_connections = 100;
.probe = {
.url = "/varnish-check.txt";
.timeout = 1s;
.interval = 5s;
.window = 5;
.threshold = 3;
}
}
# Define Server B
backend server-b {
.host = "xx.xx.xx.xx";
.port = "80";
.max_connections = 100;
.probe = {
.url = "/varnish-check.txt";
.timeout = 1s;
.interval = 5s;
.window = 5;
.threshold = 3;
}
}
# Define Server C
backend server-c {
.host = "xx.xx.xx.xx";
.port = "8080";
.max_connections = 100;
.probe = {
.url = "/varnish-check.txt";
.timeout = 1s;
.interval = 5s;
.window = 5;
.threshold = 3;
}
}
sub vcl_recv {
if (req.http.host == "example.com" || req.http.host == "example2.com") {
set req.backend_hint = server_b.backend();
}
elseif (req.http.host == "example3.com") {
set req.backend_hint = server_c.backend();
}
else {
set req.backend_hint = server_a.backend();
}
}
acl purge {
"localhost";
"127.0.0.1";
"::1";
"xx.xx.xx.xx";
"<IPv6>";
}
sub vcl_recv {
set req.http.X-Forwarded-For = regsub(req.http.X-Forwarded-For,"^([^,]+)(,[^,]+)*","\1");
if (req.method == "PURGE") {
if (!client.ip ~ purge) {
return (synth(405, "This IP is not allowed to send PURGE requests."));
}
if (req.http.X-Purge-Method == "regex") {
ban("obj.http.x-url ~ " + req.url + " && obj.http.x-host ~ " + req.http.host);
return (synth(200, "Banned"));
}
return (purge);
}
# Wordpress: don't cache these special pages
if (req.url ~ "(wp-admin|post\.php|edit\.php|wp-login)") {
return(pass);
}
# Wordpress: don't cache users who are logged-in or on password-protected pages
if (req.http.Cookie ~ "wordpress_logged_in_|resetpass|wp-postpass_") {
return(pass);
}
# Remove cookies
set req.http.Cookie = regsuball(req.http.Cookie, "comment_author_[a-zA-Z0-9_]+", "");
set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-1=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-time-1=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "wordpress_test_cookie=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "PHPSESSID=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");
# Remove proxy header (see https://httpoxy.org/#mitigate-varnish)
unset req.http.proxy;
# Normalize query arguments (sort alphabetically)
set req.url = std.querysort(req.url);
# Strip trailing ? if it exists
if (req.url ~ "\?$") {
set req.url = regsub(req.url, "\?$", "");
}
# Limit requests to the following types
if (req.method !~ "^GET|HEAD|PUT|POST|TRACE|OPTIONS|PATCH|DELETE$") {
return (pipe);
}
# Only cache GET or HEAD requests to ensure that POST requests are always passed through, along with their cookies
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# Don't cache AJAX requests
if (req.http.X-Requested-With == "XMLHttpRequest") {
return(pass);
}
# Don't cache images and PDFs
if (req.url ~ "\.(gif|jpg|jpeg|bmp|png|pdf)$") {
return(pass);
}
# Don't cache large files (zip, audio, video, etc.)
if (req.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|wmv|xz|zip)(\?.*)?$") {
return (pipe);
}
# Add support for ESI
if (req.http.Authorization) {
return (pass);
}
# Wordpress: don't cache search results
if (req.url ~ "/\?s=") {
return (pass);
}
# Wordpress: don't cache REST API (hand-rolled APIs used by custom themes)
if (req.url ~ "/shared-gc/includes/rest-api/") {
return (pass);
}
# Wordpress: don't cache anything with a cache-breaking v=<random> parameter (see gc.loadCachedJSON() JS function)
if (req.url ~ "(\?|&)v=0") {
return (pass);
}
# Don't cache the special pages we use to generate PDFs from the Wordpress catalog site
if (req.url ~ "/generate-catalog/") {
return (pass);
}
# Respect the browser's desire for a fresh copy on hard refresh. This ban will only work if there are no further URL changes (e.g. set req.url = ...) after it
if (req.http.Cache-Control == "no-cache") {
ban("req.http.host == " + req.http.host + " && req.url == " + req.url);
}
# Are there cookies left with only spaces or that are empty?
if (req.http.cookie ~ "^\s*$") {
unset req.http.cookie;
}
# Remove all cookies to enable caching
unset req.http.Cookie;
return (hash);
}
sub vcl_hash {
# Ignore marketing-related url parameters when caching urls
set req.http.newUrl = req.url;
if (req.http.newUrl ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|fbclid|cx|ie|cof|siteurl|gc_source|mkt_tok)=") {
set req.http.newUrl = regsuball(req.http.newUrl, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|fbclid|cx|ie|cof|siteurl|gc_source|mkt_tok)=([A-z0-9_\-\.%25]+)", "");
set req.http.newUrl = regsuball(req.http.newUrl, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|fbclid|cx|ie|cof|siteurl|gc_source|mkt_tok)=([A-z0-9_\-\.%25]+)", "?");
set req.http.newUrl = regsub(req.http.newUrl, "\?&", "?");
set req.http.newUrl = regsub(req.http.newUrl, "\?$", "");
}
# Ignore hashes when caching urls
if (req.http.newUrl ~ "\#") {
set req.http.newUrl = regsub(req.http.newUrl, "\#.*$", "");
}
# Default vcl_hash, except replaced "req.url" with "req.http.newUrl"
hash_data(req.http.newUrl);
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
return (lookup);
}
sub vcl_backend_response {
set beresp.http.x-url = bereq.url;
set beresp.http.x-host = bereq.http.host;
# Set the TTL for the cache to thirty days and the grace period to twelve hours
set beresp.ttl = 30d;
set beresp.grace = 12h;
set beresp.keep = 24h;
# Set different TTLs for other hosts
# if (bereq.url ~ "(example.com|secondexample.com)") {
# set beresp.ttl = 30d;
#}
# Set 301 and 302 as uncacheable
if (beresp.status == 301 || beresp.status == 302) {
set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
# Don't cache redirects
set beresp.uncacheable = true;
}
# Cache 404 responses for five minutes (can be cleared by hard refresh)
if (beresp.status == 403 || beresp.status == 404)
{
set beresp.ttl = 5m;
}
# Check for the response status of background fetches from backend, and return (abandon) if the response is a “5XX” errors
if (bereq.is_bgfetch && beresp.status >= 500 && beresp.status <= 599) {
return (abandon);
}
}
sub vcl_deliver {
unset resp.http.x-url;
unset resp.http.x-host;
# Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
if (obj.hits > 0) {
set resp.http.X-Cache = "HIT";
} else {
set resp.http.X-Cache = "MISS";
}
set resp.http.X-Cache-Hits = obj.hits;
# Remove headers to improve security
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.X-Powered-By;
unset resp.http.Server;
}
sub vcl_init {
return (ok);
}
sub vcl_fini {
return (ok);
}
sub vcl_hit {
# If the object has a TTL equal to or greater than 0s, deliver it from the cache
if (obj.ttl >= 0s) {
return (deliver);
}
# Check whether Grace Mode is necessary
if (std.healthy(req.backend_hint)) {
if (obj.ttl + 10s > 0s) {
set req.http.grace = "normal(limited)";
return (deliver);
} else {
# If Grace Mode is not necessary, fetch a fresh object from the backend
return(miss);
}
} else {
# If Grace Mode is necessary (i.e. the backend is down), enter grace mode
if (obj.ttl + obj.grace > 0s) {
set req.http.grace = "full";
return (deliver);
} else {
# If there is no Grace Mode object available, send to the backend
return (miss);
}
}
}
This doesn't seem to work, but I can't see why. If a backend server goes down, the Varnish server quickly returns a 503 or other error. Any pointers as to what I've got wrong would be appreciated.
Relatedly, I really like the ability to clear the cache with a hard browser refresh. But it strikes me that it would be great to be able to negate that rule if the backend is down. Any idea how I go about that?
And, of course, if there are any obvious errors in here, I'd love to hear about them.
Thanks!

Answer 1

Apparently you have some logic in vcl_hit that inspects the backend health and interferes with the TTL and grace value.
However, there's a simpler way of handling what is called Stale If Error. Here's the VCL code you need:
vcl 4.1;
import std;
backend default {
.host = "xx.xx.xx.xx";
.port = "80";
.max_connections = 100;
.probe = {
.url = "/varnish-check.txt";
.timeout = 1s;
.interval = 5s;
.window = 5;
.threshold = 3;
}
}
sub vcl_recv {
if (std.healthy(req.backend_hint)) {
set req.grace = 10s;
}
}
sub vcl_backend_response {
set beresp.grace = 24h;
}
Total object lifetime
To understand the logic behind the VCL code, you need to understand how Varnish determines the object lifetime.
The total object lifetime is the sum of the following elements:
Total object lifetime = TTL + grace +keep
As long as the sum of these elements is greater than zero, the object is kept around in cache. That doesn't mean revalidation doesn't happen.
If the remaining TTL drops below zero, Varnish attempts to connect to the origin server. But if there's grace left, it will do this asynchronously while it's serving the stale content to the client.
If the backend is down, the stale content is still served.
If the object has expired and is out of grace, synchronous revalidation happens. If there is some keep time left, the potential ETag and Last-Modified headers of the expired object are used and converted into If-None-Match and If-Modified-Since backend request headers.
Conditional requests are the only real feature of keep time, because synchronous revalidation puts client requests in the queue, unlike grace mode.
How to leverage grace mode for stale if error
Grace mode is Varnish's implementation of stale while revalidate and can be set in VCL using beresp.grace but also via Cache-Control: stale-while-revalidate=3600.
What we're trying to do here is take advantage of grace mode's capability to send stale content to the client while revalidating asynchronously.
As the VCL example shows we're saving the object with a grace value of 24 hours. However, when requesting the object, we're only using 10 seconds of grace as long as the backend is healthy.
If it turns out the backend doesn't respond, the original grace of 24 hours is used.
Using grace for stale if error works, but it's a bit of a hack
See https://www.youtube.com/watch?v=51WUTB1cUeM for a 2-minute video about grace mode.
Using vmod_stale
Varnish also has a proper implementation of stale if error, but it's only available in Varnish Enterprise.
We specifically built vmod_stale to handle situations where backends are down.
Here's some example VCL code that uses vmod_stale:
vcl 4.1;
import stale;
backend default {
.host = "xx.xx.xx.xx";
.port = "80";
.max_connections = 100;
.probe = {
.url = "/varnish-check.txt";
.timeout = 1s;
.interval = 5s;
.window = 5;
.threshold = 3;
}
}
sub stale_if_error {
if (beresp.status >= 500 && stale.exists()) {
stale.revive(20m, 1h);
stale.deliver();
return (abandon);
}
}
sub vcl_backend_response {
call stale_if_error;
}
sub vcl_backend_error {
call stale_if_error;
}
This VCL example leverages stale.revive() to set new values for TTL and grace while respecting the original total expiration time.
If the new TTL and grace time in combination with the existing keep time exceed the total life time of the object, the overflow is deducted from the keep time.
See https://www.youtube.com/watch?v=6LY4Idt1e2Q for a video about this VMOD.