Helm not Importing file correctly - go

Question

I am creating an Helm Chart and I am having problems when it comes to importing files:
apiVersion: v1
kind: ConfigMap
metadata:
name: vcl-template
namespace: {{.Release.Namespace}}
data:
{{- (.Files.Glob "config/varnish/default.vcl.tmpl").AsConfig | nindent 2 }}
{{- (.Files.Glob "config/varnish/nginx.conf").AsConfig | nindent 2 }}
This imports the file config/varnish/nginx.conf just fine but the file config/varnish/default.vcl.tmpl is imported with \n instead of newlines, so the data on the ConfigMap gets all buggy:
apiVersion: v1
kind: ConfigMap
metadata:
name: vcl-template
namespace: default
data:
default.vcl.tmpl: "vcl 4.0;\n\nimport std;\nimport directors;\n\n{{ range .Frontends
}}\nbackend {{ .Name }} {\n .host = \"{{ .Host }}\";\n .port = \"{{ .Port
}}\";\n}\n{{- end }}\n\n{{ range .Backends }}\nbackend be-{{ .Name }} {\n .host
= \"{{ .Host }}\";\n .port = \"{{ .Port }}\";\n}\n{{- end }}\n\nacl purge {\n
\ \"127.0.0.1\";\n \"localhost\";\n \"::1\";\n {{- range .Frontends }}\n
\ \"{{ .Host }}\";\n {{- end }}\n {{- range .Backends }}\n \"{{ .Host
}}\";\n {{- end }}\n}\n\nsub vcl_init {\n new cluster = directors.hash();\n\n
\ {{ range .Frontends -}}\n cluster.add_backend({{ .Name }}, 1);\n {{ end
}}\n\n new lb = directors.round_robin();\n\n {{ range .Backends -}}\n lb.add_backend(be-{{
.Name }});\n {{ end }}\n}\n\nsub vcl_recv {\n\n unset req.http.x-cache;\n
\ set req.backend_hint = cluster.backend(req.url);\n set req.http.x-shard =
req.backend_hint;\n if (req.http.x-shard != server.identity) {\n return(pass);\n
\ }\n set req.backend_hint = lb.backend();\n\n if (req.method == \"PURGE\")
{\n if (client.ip !~ purge) {\n return (synth(405, \"Method not
allowed\"));\n }\n # To use the X-Pool header for purging varnish
during automated deployments, make sure the X-Pool header\n # has been added
to the response in your backend server config. This is used, for example, by the\n
\ # capistrano-magento2 gem for purging old content from varnish during it's
deploy routine.\n if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool)
{\n return (synth(400, \"X-Magento-Tags-Pattern or X-Pool header required\"));\n
\ }\n if (req.http.X-Magento-Tags-Pattern) {\n ban(\"obj.http.X-Magento-Tags
~ \" + req.http.X-Magento-Tags-Pattern);\n }\n if (req.http.X-Pool)
{\n ban(\"obj.http.X-Pool ~ \" + req.http.X-Pool);\n }\n return
(synth(200, \"Purged\"));\n }\n\n if (req.method != \"GET\" &&\n req.method
!= \"HEAD\" &&\n req.method != \"PUT\" &&\n req.method != \"POST\"
&&\n req.method != \"TRACE\" &&\n req.method != \"OPTIONS\" &&\n req.method
!= \"DELETE\") {\n /* Non-RFC2616 or CONNECT which is weird. */\n return
(pipe);\n }\n\n # We only deal with GET and HEAD by default\n if (req.method
!= \"GET\" && req.method != \"HEAD\") {\n return (pass);\n }\n\n #
Bypass shopping cart, checkout and search requests\n if (req.url ~ \"/checkout\"
|| req.url ~ \"/catalogsearch\") {\n return (pass);\n }\n\n # Bypass
admin\n if (req.url ~ \"^/admin($|/.*)\") {\n return (pass);\n }\n\n
\ # Bypass health check requests\n if (req.url ~ \"/pub/health_check.php\")
{\n return (pass);\n }\n\n # Set initial grace period usage status\n
\ set req.http.grace = \"none\";\n\n # normalize url in case of leading HTTP
scheme and domain\n set req.url = regsub(req.url, \"^http[s]?://\", \"\");\n\n
\ # collect all cookies\n std.collect(req.http.Cookie);\n\n # Compression
filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression\n if (req.http.Accept-Encoding)
{\n if (req.url ~ \"\\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$\")
{\n # No point in compressing these\n unset req.http.Accept-Encoding;\n
\ } elsif (req.http.Accept-Encoding ~ \"gzip\") {\n set req.http.Accept-Encoding
= \"gzip\";\n } elsif (req.http.Accept-Encoding ~ \"deflate\" && req.http.user-agent
!~ \"MSIE\") {\n set req.http.Accept-Encoding = \"deflate\";\n }
else {\n # unknown algorithm\n unset req.http.Accept-Encoding;\n
\ }\n }\n\n # Remove all marketing get parameters to minimize the cache
objects\n if (req.url ~ \"(\\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=\")
{\n set req.url = regsuball(req.url, \"(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?\",
\"\");\n set req.url = regsub(req.url, \"[?|&]+$\", \"\");\n }\n\n #
Static files caching\n if (req.url ~ \"^/(pub/)?(media|static)/\") {\n return
(pass);\n }\n\n return (hash);\n}\n\nsub vcl_hash {\n if (req.http.cookie
~ \"X-Magento-Vary=\") {\n hash_data(regsub(req.http.cookie, \"^.*?X-Magento-Vary=([^;]+);*.*$\",
\"\\1\"));\n }\n\n # For multi site configurations to not cache each other's
content\n if (req.http.host) {\n hash_data(req.http.host);\n } else
{\n hash_data(server.ip);\n }\n\n if (req.url ~ \"/graphql\") {\n call
process_graphql_headers;\n }\n\n # To make sure http users don't see ssl warning\n
\ if (req.http.X-Forwarded-Proto) {\n hash_data(req.http.X-Forwarded-Proto);\n
\ }\n \n}\n\nsub process_graphql_headers {\n if (req.http.Store) {\n hash_data(req.http.Store);\n
\ }\n if (req.http.Content-Currency) {\n hash_data(req.http.Content-Currency);\n
\ }\n}\n\nsub vcl_backend_response {\n\n set beresp.grace = 3d;\n\n if (beresp.http.content-type
~ \"text\") {\n set beresp.do_esi = true;\n }\n\n if (bereq.url ~ \"\\.js$\"
|| beresp.http.content-type ~ \"text\") {\n set beresp.do_gzip = true;\n
\ }\n\n if (beresp.http.X-Magento-Debug) {\n set beresp.http.X-Magento-Cache-Control
= beresp.http.Cache-Control;\n }\n\n # cache only successfully responses and
404s\n if (beresp.status != 200 && beresp.status != 404) {\n set beresp.ttl
= 0s;\n set beresp.uncacheable = true;\n return (deliver);\n }
elsif (beresp.http.Cache-Control ~ \"private\") {\n set beresp.uncacheable
= true;\n set beresp.ttl = 86400s;\n return (deliver);\n }\n\n
\ # validate if we need to cache it and prevent from setting cookie\n if (beresp.ttl
> 0s && (bereq.method == \"GET\" || bereq.method == \"HEAD\")) {\n unset
beresp.http.set-cookie;\n }\n\n # If page is not cacheable then bypass varnish
for 2 minutes as Hit-For-Pass\n if (beresp.ttl <= 0s ||\n beresp.http.Surrogate-control
~ \"no-store\" ||\n (!beresp.http.Surrogate-Control &&\n beresp.http.Cache-Control
~ \"no-cache|no-store\") ||\n beresp.http.Vary == \"*\") {\n # Mark
as Hit-For-Pass for the next 2 minutes\n set beresp.ttl = 120s;\n set
beresp.uncacheable = true;\n }\n\n return (deliver);\n}\n\nsub vcl_deliver
{\n if (resp.http.X-Magento-Debug) {\n if (resp.http.x-varnish ~ \" \")
{\n set resp.http.X-Magento-Cache-Debug = \"HIT\";\n set resp.http.Grace
= req.http.grace;\n } else {\n set resp.http.X-Magento-Cache-Debug
= \"MISS\";\n }\n } else {\n unset resp.http.Age;\n }\n\n #
Not letting browser to cache non-static files.\n if (resp.http.Cache-Control
!~ \"private\" && req.url !~ \"^/(pub/)?(media|static)/\") {\n set resp.http.Pragma
= \"no-cache\";\n set resp.http.Expires = \"-1\";\n set resp.http.Cache-Control
= \"no-store, no-cache, must-revalidate, max-age=0\";\n }\n\n unset resp.http.X-Magento-Debug;\n
\ unset resp.http.X-Magento-Tags;\n unset resp.http.X-Powered-By;\n unset
resp.http.Server;\n unset resp.http.X-Varnish;\n unset resp.http.Via;\n unset
resp.http.Link;\n}\n\nsub vcl_hit {\n if (obj.ttl >= 0s) {\n # Hit within
TTL period\n return (deliver);\n }\n if (std.healthy(req.backend_hint))
{\n if (obj.ttl + 300s > 0s) {\n # Hit after TTL expiration, but
within grace period\n set req.http.grace = \"normal (healthy server)\";\n
\ return (deliver);\n } else {\n # Hit after TTL and
grace expiration\n return (miss);\n }\n } else {\n #
server is not healthy, retrieve from cache\n set req.http.grace = \"unlimited
(unhealthy server)\";\n return (deliver);\n }\n}\n"
nginx.conf: |
worker_processes auto;
events {
worker_connections 1024;
}
pcre_jit on;
error_log /var/log/nginx/error.log warn;
include /etc/nginx/modules/*.conf;
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
server_tokens off;
client_max_body_size 15m;
keepalive_timeout 30;
sendfile on;
tcp_nodelay on;
gzip_vary on;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
include /etc/nginx/conf.d/*.conf;
}
nginx.conf:
worker_processes auto;
events {
worker_connections 1024;
}
pcre_jit on;
error_log /var/log/nginx/error.log warn;
include /etc/nginx/modules/*.conf;
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
server_tokens off;
client_max_body_size 15m;
keepalive_timeout 30;
sendfile on;
tcp_nodelay on;
gzip_vary on;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
include /etc/nginx/conf.d/*.conf;
}
default.vcl.tmpl:
vcl 4.0;
import std;
import directors;
{{ range .Frontends }}
backend {{ .Name }} {
.host = "{{ .Host }}";
.port = "{{ .Port }}";
}
{{- end }}
{{ range .Backends }}
backend be-{{ .Name }} {
.host = "{{ .Host }}";
.port = "{{ .Port }}";
}
{{- end }}
acl purge {
"127.0.0.1";
"localhost";
"::1";
{{- range .Frontends }}
"{{ .Host }}";
{{- end }}
{{- range .Backends }}
"{{ .Host }}";
{{- end }}
}
sub vcl_init {
new cluster = directors.hash();
{{ range .Frontends -}}
cluster.add_backend({{ .Name }}, 1);
{{ end }}
new lb = directors.round_robin();
{{ range .Backends -}}
lb.add_backend(be-{{ .Name }});
{{ end }}
}
sub vcl_recv {
unset req.http.x-cache;
set req.backend_hint = cluster.backend(req.url);
set req.http.x-shard = req.backend_hint;
if (req.http.x-shard != server.identity) {
return(pass);
}
set req.backend_hint = lb.backend();
if (req.method == "PURGE") {
if (client.ip !~ purge) {
return (synth(405, "Method not allowed"));
}
# To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
# has been added to the response in your backend server config. This is used, for example, by the
# capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
}
if (req.http.X-Magento-Tags-Pattern) {
ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
}
if (req.http.X-Pool) {
ban("obj.http.X-Pool ~ " + req.http.X-Pool);
}
return (synth(200, "Purged"));
}
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# Bypass shopping cart, checkout and search requests
if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") {
return (pass);
}
# Bypass admin
if (req.url ~ "^/admin($|/.*)") {
return (pass);
}
# Bypass health check requests
if (req.url ~ "/pub/health_check.php") {
return (pass);
}
# Set initial grace period usage status
set req.http.grace = "none";
# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://", "");
# collect all cookies
std.collect(req.http.Cookie);
# Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
if (req.http.Accept-Encoding) {
if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
# No point in compressing these
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
set req.http.Accept-Encoding = "deflate";
} else {
# unknown algorithm
unset req.http.Accept-Encoding;
}
}
# Remove all marketing get parameters to minimize the cache objects
if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
set req.url = regsub(req.url, "[?|&]+$", "");
}
# Static files caching
if (req.url ~ "^/(pub/)?(media|static)/") {
return (pass);
}
return (hash);
}
sub vcl_hash {
if (req.http.cookie ~ "X-Magento-Vary=") {
hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
}
# For multi site configurations to not cache each other's content
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
if (req.url ~ "/graphql") {
call process_graphql_headers;
}
# To make sure http users don't see ssl warning
if (req.http.X-Forwarded-Proto) {
hash_data(req.http.X-Forwarded-Proto);
}
}
sub process_graphql_headers {
if (req.http.Store) {
hash_data(req.http.Store);
}
if (req.http.Content-Currency) {
hash_data(req.http.Content-Currency);
}
}
sub vcl_backend_response {
set beresp.grace = 3d;
if (beresp.http.content-type ~ "text") {
set beresp.do_esi = true;
}
if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
set beresp.do_gzip = true;
}
if (beresp.http.X-Magento-Debug) {
set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
}
# cache only successfully responses and 404s
if (beresp.status != 200 && beresp.status != 404) {
set beresp.ttl = 0s;
set beresp.uncacheable = true;
return (deliver);
} elsif (beresp.http.Cache-Control ~ "private") {
set beresp.uncacheable = true;
set beresp.ttl = 86400s;
return (deliver);
}
# validate if we need to cache it and prevent from setting cookie
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
unset beresp.http.set-cookie;
}
# If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
if (beresp.ttl <= 0s ||
beresp.http.Surrogate-control ~ "no-store" ||
(!beresp.http.Surrogate-Control &&
beresp.http.Cache-Control ~ "no-cache|no-store") ||
beresp.http.Vary == "*") {
# Mark as Hit-For-Pass for the next 2 minutes
set beresp.ttl = 120s;
set beresp.uncacheable = true;
}
return (deliver);
}
sub vcl_deliver {
if (resp.http.X-Magento-Debug) {
if (resp.http.x-varnish ~ " ") {
set resp.http.X-Magento-Cache-Debug = "HIT";
set resp.http.Grace = req.http.grace;
} else {
set resp.http.X-Magento-Cache-Debug = "MISS";
}
} else {
unset resp.http.Age;
}
# Not letting browser to cache non-static files.
if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
set resp.http.Pragma = "no-cache";
set resp.http.Expires = "-1";
set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
}
unset resp.http.X-Magento-Debug;
unset resp.http.X-Magento-Tags;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
}
sub vcl_hit {
if (obj.ttl >= 0s) {
# Hit within TTL period
return (deliver);
}
if (std.healthy(req.backend_hint)) {
if (obj.ttl + 300s > 0s) {
# Hit after TTL expiration, but within grace period
set req.http.grace = "normal (healthy server)";
return (deliver);
} else {
# Hit after TTL and grace expiration
return (miss);
}
} else {
# server is not healthy, retrieve from cache
set req.http.grace = "unlimited (unhealthy server)";
return (deliver);
}
}
How come that the second file is not imported correctly? Latest Helm version and latest Go version.
Anyone has any ideas? The encoding of both files on VSCode shows as UTF8.

Answer 1

They're actually equivalent from YAML's PoV, just not as pretty, but most important for your specific case it's because yaml cannot represent trailing whitespace without quoting it, which is what it did due to line 164 of your .tmpl file, as seen by the \n \n in:
\ }\n \n}\n\nsub process_graphql_headers {\n if (req.http.Store) {\n hash_data(req.http.Store);\n
$ sed -ne 164p default.vcl.tmpl | xxd
00000000: 2020 2020 0a .
turning on "strip trailing whitespace" in your editor will help that, or for this specific case you can just fix line 164