Magento 2 varnish not caching, delivers new cookie each time - magento

Question

I'm trying to get Varnish to cache a magento store but it keeps setting different cookies each time I curl it:
$ curl -IL -X GET https://myurl.com/ |grep cookie
set-cookie: store=default; expires=Sat, 17-Apr-2021 19:51:22 GMT; Max-Age=31536000; path=/index.php/; HttpOnly
set-cookie: PHPSESSID=j5uhb0oe5qh8d212j12sfcnsaa; expires=Fri, 17-Apr-2020 20:51:22 GMT; Max-Age=3600; path=/; domain=myurl.com; HttpOnly
$ curl -IL -X GET https://myurl.com/ |grep cookie
set-cookie: store=default; expires=Sat, 17-Apr-2021 19:51:26 GMT; Max-Age=31536000; path=/index.php/; HttpOnly
set-cookie: PHPSESSID=dg09e4uaj9kiqo37rp4pk2g8co; expires=Fri, 17-Apr-2020 20:51:26 GMT; Max-Age=3600; path=/; domain=myurl.com; HttpOnly
Website is loading pretty slow.
My varnish config:
import std;
# The minimal Varnish version is 5.0
# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https'
backend default {
.host = "localhost";
.port = "8080";
.first_byte_timeout = 600s;
.probe = {
.url = "/health_check.php";
.timeout = 2s;
.interval = 5s;
.window = 10;
.threshold = 5;
}
}
acl purge {
"localhost";
"172.25.0.0"/16;
}
sub vcl_recv {
# set req.backend_hint = vdir.backend(); # send all traffic to the vdir director
if (req.method == "PURGE") {
if (client.ip !~ purge) {
return (synth(405, "Method not allowed"));
}
# To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
# has been added to the response in your backend server config. This is used, for example, by the
# capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
}
if (req.http.X-Magento-Tags-Pattern) {
ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
}
if (req.http.X-Pool) {
ban("obj.http.X-Pool ~ " + req.http.X-Pool);
}
# If all Tags should be purged clear
# # ban everything to catch assets as well
if (req.http.X-Magento-Tags-Pattern == ".*") {
ban("req.url ~ .*");
}
return (synth(200, "Purged Magento"));
}
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
if (req.url ~ "/healthcheck") {
return (pass);
}
# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# Bypass shopping cart, checkout and search requests
if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") {
return (pass);
}
# Bypass health check requests
if (req.url ~ "/health_check.php") {
return (pass);
}
# Set initial grace period usage status
set req.http.grace = "none";
# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://", "");
# collect all cookies
std.collect(req.http.Cookie);
# Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
if (req.http.Accept-Encoding) {
if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
# No point in compressing these
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
set req.http.Accept-Encoding = "deflate";
} else {
# unkown algorithm
unset req.http.Accept-Encoding;
}
}
# Remove Google gclid parameters to minimize the cache objects
set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
# Static files caching
if (req.url ~ "^/(pub/)?(media|static)/") {
# Static files should not be cached by default
#return (pass);
return (hash);
# But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
unset req.http.Https;
unset req.http.X-Forwarded-Proto;
unset req.http.Cookie;
}
return (hash);
}
sub vcl_hash {
if (req.http.cookie ~ "X-Magento-Vary=") {
hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
}
# For multi site configurations to not cache each other's content
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
# To make sure http users don't see ssl warning
if (req.http.X-Forwarded-Proto) {
hash_data(req.http.X-Forwarded-Proto);
}
}
sub vcl_backend_response {
set beresp.grace = 3d;
if (beresp.http.content-type ~ "text") {
set beresp.do_esi = true;
}
if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
set beresp.do_gzip = true;
}
if (beresp.http.X-Magento-Debug) {
set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
}
# cache only successfully responses and 404s
#if (beresp.status != 200 && beresp.status != 404)) {
# don't cache 404 nor 300 nor 500
if (beresp.status != 200) {
set beresp.ttl = 0s;
set beresp.uncacheable = true;
return (deliver);
} elsif (beresp.http.Cache-Control ~ "private") {
set beresp.uncacheable = true;
set beresp.ttl = 86400s;
return (deliver);
}
# validate if we need to cache it and prevent from setting cookie
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
unset beresp.http.set-cookie;
}
# If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
if (beresp.ttl <= 0s ||
beresp.http.Surrogate-control ~ "no-store" ||
(!beresp.http.Surrogate-Control &&
beresp.http.Cache-Control ~ "no-cache|no-store") ||
beresp.http.Vary == "*") {
# Mark as Hit-For-Pass for the next 2 minutes
set beresp.ttl = 120s;
set beresp.uncacheable = true;
}
return (deliver);
}
sub vcl_deliver {
if (resp.http.X-Magento-Debug) {
if (resp.http.x-varnish ~ " ") {
set resp.http.X-Magento-Cache-Debug = "HIT";
set resp.http.Grace = req.http.grace;
} else {
set resp.http.X-Magento-Cache-Debug = "MISS";
}
} else {
unset resp.http.Age;
}
set resp.http.X-Test = "YEAH";
if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
set resp.http.X-Cache = "HIT";
} else {
set resp.http.X-Cache = "MISS";
}
# Please note that obj.hits behaviour changed in 4.0, now it counts per objecthead, not per object
# and obj.hits may not be reset in some cases where bans are in use. See bug 1492 for details.
# So take hits with a grain of salt
set resp.http.X-Cache-Hits = obj.hits;
# Not letting browser to cache non-static files.
if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
set resp.http.Pragma = "no-cache";
set resp.http.Expires = "-1";
set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
}
unset resp.http.X-Magento-Debug;
unset resp.http.X-Magento-Tags;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
}
sub vcl_hit {
if (obj.ttl >= 0s) {
# Hit within TTL period
return (deliver);
}
if (std.healthy(req.backend_hint)) {
if (obj.ttl + 300s > 0s) {
# Hit after TTL expiration, but within grace period
set req.http.grace = "normal (healthy server)";
return (deliver);
} else {
# Hit after TTL and grace expiration
return (miss);
}
} else {
# server is not healthy, retrieve from cache
set req.http.grace = "unlimited (unhealthy server)";
return (deliver);
}
}
Caché:
$ bin/magento cache:status
Current status:
config: 1
layout: 0
block_html: 0
collections: 1
reflection: 1
db_ddl: 1
eav: 1
customer_notification: 1
config_integration: 1
config_integration_api: 1
full_page: 0
translate: 1
config_webservice: 1
vertex: 0
wp_gtm_categories: 1
php bin/magento config:show |grep -i cache
system/full_page_cache/varnish/access_list - localhost
system/full_page_cache/varnish/backend_host - localhost
system/full_page_cache/varnish/backend_port - 8080
system/full_page_cache/varnish/grace_period - 300
system/full_page_cache/caching_application - 2
Is this expected? site loads extremely slow, even if I reload it over and over again
Thanks for your time, gentleman.

Answer 1

Based on de VCL code, it doesn't look like Varnish is removing the PHPSESSID and the store cookies. One way to be sure, is by running the following varnishlog command on your Varnish server:
varnishlog -g request -i ReqUrl -I ReqUnset:cookie -I ReqHeader:Cookie -I bereqheader:Cookie -I berequnset:cookie -q "ReqUrl eq '/'"
This will look at requests from requests to the homepage and will list the following things:
URL
Cookies set by the client as a request header
A copy of the cookies, set as a backend request header
Possible cookies being unset in the client part of the VCL (e.g. in vcl_recv)
Possible cookies being unset in the backend part of the VCL (e.g. in vcl_backend_request)
This will give you a clear indication whether or not cookies are remove by Varnish. You can replace the cookie header filter with any other header potentially being removed.
If you want to know which VCL flow is run, you can add -i "VCL_*" and then you'll see if it is a HIT, a MISS, or a deliberate PASS.
Once you've done the necessary debugging, you'll know if it is a Varnish issue, or a PHP issue.

Answer 2

check for X-Magento-Vary cookie. It might be on a Ajax response and this cookie is in hash routine