Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Varnish in-depth training - Symfony Live Berlin 2017

Thijs Feryn
October 25, 2017

Varnish in-depth training - Symfony Live Berlin 2017

Thijs Feryn

October 25, 2017
Tweet

More Decks by Thijs Feryn

Other Decks in Technology

Transcript

  1. varnishd VCL file Compiled & linked as shared object Client

    HTTP Backend HTTP Memory Logs Writes verbose “transaction” logs to memory varnishlog varnishtop Read memory logs, displays information varnishncsa varnishadm Allow basic management tasks Custom protocol
  2. $ apt-cache policy varnish varnish: Installed: 5.2.0-1~stretch Candidate: 5.2.0-1~stretch Version

    table: *** 5.2.0-1~stretch 500 500 https://packagecloud.io/varnishcache/varnish5/debian stretch/main amd64 Packages 100 /var/lib/dpkg/status 5.1.3-1~stretch 500 500 https://packagecloud.io/varnishcache/varnish5/debian stretch/main amd64 Packages 5.1.2-1~stretch 500 500 https://packagecloud.io/varnishcache/varnish5/debian stretch/main amd64 Packages 5.0.0-7+deb9u1 500 500 http://deb.debian.org/debian stretch/main amd64 Packages 500 http://security.debian.org stretch/updates/main amd64 Packages
  3. [Unit] Description=Varnish Cache, a high-performance HTTP accelerator [Service] Type=forking #

    Maximum number of open files (for ulimit -n) LimitNOFILE=131072 # Locked shared memory - should suffice to lock the shared memory log # (varnishd -l argument) # Default log size is 80MB vsl + 1M vsm + header -> 82MB # unit is bytes LimitMEMLOCK=85983232 # On systemd >= 228 enable this to avoid "fork failed" on reload. #TasksMax=infinity # Maximum size of the corefile. LimitCORE=infinity # Set WARMUP_TIME to force a delay in reload-vcl between vcl.load and vcl.use # This is useful when backend probe definitions need some time before declaring # configured backends healthy, to avoid routing traffic to a non-healthy backend. #WARMUP_TIME=0 ExecStart=/usr/sbin/varnishd -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/ secret -s malloc,256m ExecReload=/usr/share/varnish/reload-vcl [Install] WantedBy=multi-user.target Debian Stretch with systemd
  4. DAEMON_OPTS="-a :80 \ -T localhost:6082 \ -f /etc/varnish/default.vcl \ -S

    /etc/varnish/secret \ -s malloc,256m" Ubuntu Trusty without systemd
  5. -a: binding address & port -T: admin binding -f: VCL

    file -S: secret file -s: storage -j: jailing -l: shared memory log size -t: default TTL -p: runtime parameters Options
  6. DAEMON_OPTS="-j unix,user=www-data \ -a main=0000:80 \ -a proxy=0.0.0.0:81,PROXY \ -T

    localhost:6082 \ -f /etc/varnish/default.vcl \ -S /etc/varnish/secret \ -l 100m,10m \ -t 60 \ -p feature=+esi_disable_xml_check \ -p connect_timeout=20s \ -p first_byte_timeout=100s \ -p between_bytes_timeout=5s \ -s malloc,3g"
  7. 80

  8. DAEMON_OPTS="-j unix,user=www-data \ -a main=0.0.0.0:80 \ -a proxy=0.0.0.0:81,PROXY \ -T

    localhost:6082 \ -b 127.0.0.1:8080 \ -S /etc/varnish/secret \ -s malloc,3g" Can't have -f In config file
  9. Share the cache User Nginx (80) Nginx (80) 1 2

    3 4 5 6 HAProxy (80) Varnish ( 6080 & 6081) HAProxy (80) Varnish ( 6080 & 6081)
  10. Share the cache 2 User HAProxy (80) Varnish ( 6080

    & 6081) Nginx (80) Nginx (80) 1 5 3 4 2 6 HAProxy (80) Varnish ( 6080 & 6081)
  11. vcl 4.0; import std; backend nginx { .host = "nginx";

    .port = "80"; } backend nginx2 { .host = "nginx2"; .port = "80"; } backend varnish { .host = "varnish"; .port = "6080"; } backend varnish2 { .host = "varnish2"; .port = "6080"; } sub vcl_recv { if(server.hostname == "varnish") { if(std.port(server.ip) == 6080) { set req.backend_hint = nginx; set req.http.x-esi = "yes"; } else { set req.backend_hint = varnish2; set req.http.x-esi = "no"; } } else { if(std.port(server.ip) == 6080) { set req.backend_hint = nginx2; set req.http.x-esi = "yes"; } else { set req.backend_hint = varnish; set req.http.x-esi = "no"; } } } sub vcl_backend_response { if (bereq.http.x-esi == "yes" && beresp.http.Surrogate-Control ~ "ESI/1.0") { unset beresp.http.Surrogate-Control; set beresp.do_esi = true; } }
  12. sub vcl_recv { if(server.hostname == "varnish") { if(std.port(server.ip) == 6080)

    { set req.backend_hint = nginx; set req.http.x-esi = "yes"; } else { set req.backend_hint = varnish2; set req.http.x-esi = "no"; } } else { if(std.port(server.ip) == 6080) { set req.backend_hint = nginx2; set req.http.x-esi = "yes"; } else { set req.backend_hint = varnish; set req.http.x-esi = "no"; } } }
  13. SetEnvIf X-Forwarded-Proto "https" HTTPS=on Header append Vary: X-Forwarded-Proto <IfModule mod_rewrite.c>

    RewriteEngine on RewriteCond %{HTTPS} !=on RewriteCond %{HTTP:X-Forwarded-Proto} !https [NC] RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] </IfModule>
  14. $ docker-compose up $ docker-compose down $ docker-compose up -d

    $ docker-compose logs $ docker-compose logs -f $ docker-compose logs varnish $ docker-compose logs -f varnish
  15. HAProxy (80 & 81) Varnish (6080, 6081, 6082, 6083) Nginx

    (80) PHP-FPM (9000) Varnish 2 (6080, 6081, 6082, 6083) Nginx 2 (80) PHP-FPM 2 (9000) MySQL (3306) Redis (6379) Demo setup
  16. ✓sflive-varnish2: 172.18.0.10 ✓sflive-varnish: 172.18.0.8 ✓sflive-nginx2: 172.18.0.6 ✓sflive-nginx: 172.18.0.2 ✓sflive-php2: 172.18.0.5

    ✓sflive-php: 172.18.0.4 ✓sflive-haproxy: 172.18.0.3 ✓sflive-redis: 172.18.0.7 ✓sflive-mysql: 172.18.0.9 Demo setup IPs
  17. FROM debian:stretch as builder WORKDIR /usr/lib/varnish/vmods/ RUN apt-get update \

    && apt-get install -y curl \ && curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/ script.deb.sh | bash \ && apt-get install -y unzip automake varnish=5.2.0-1~stretch varnish- dev=5.2.0-1~stretch build-essential libtool docutils-common libmhash-dev --allow- unauthenticated \ && curl -O https://download.varnish-software.com/varnish-modules/varnish- modules-0.12.1.tar.gz \ && tar xvzf varnish-modules-0.12.1.tar.gz \ && (cd varnish-modules-0.12.1 && ./configure && make && make install) \ && curl -O -L https://github.com/gquintard/libvmod-accept/archive/master.zip \ && unzip master.zip \ && (cd libvmod-accept-master && ./autogen.sh && ./configure && make && make install) \ && curl -O -L https://github.com/varnish/libvmod-digest/archive/master.zip \ && unzip master.zip \ && (cd libvmod-digest-master && ./autogen.sh && ./configure && make && make install) FROM debian:stretch WORKDIR /etc/varnish EXPOSE 6080 EXPOSE 6081 EXPOSE 6082 EXPOSE 6083 ENV SIZE 100m ENV SECRET E29F6DE6-3803-449E-8A01-AA537A8715B1 ARG BACKEND=nginx
  18. FROM debian:stretch as builder WORKDIR /usr/lib/varnish/vmods/ RUN apt-get update \

    && apt-get install -y curl \ && curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/ script.deb.sh | bash \ && apt-get install -y unzip automake varnish=5.2.0-1~stretch varnish- dev=5.2.0-1~stretch build-essential libtool docutils-common libmhash-dev --allow- unauthenticated \ && curl -O https://download.varnish-software.com/varnish-modules/varnish- modules-0.12.1.tar.gz \ && tar xvzf varnish-modules-0.12.1.tar.gz \ && (cd varnish-modules-0.12.1 && ./configure && make && make install) \ && curl -O -L https://github.com/gquintard/libvmod-accept/archive/master.zip \ && unzip master.zip \ && (cd libvmod-accept-master && ./autogen.sh && ./configure && make && make install) \ && curl -O -L https://github.com/varnish/libvmod-digest/archive/master.zip \ && unzip master.zip \ && (cd libvmod-digest-master && ./autogen.sh && ./configure && make && make install) FROM debian:stretch WORKDIR /etc/varnish EXPOSE 6080 EXPOSE 6081 EXPOSE 6082 EXPOSE 6083 ENV SIZE 100m ENV SECRET E29F6DE6-3803-449E-8A01-AA537A8715B1 ARG BACKEND=nginx
  19. FROM debian:stretch as builder WORKDIR /usr/lib/varnish/vmods/ RUN apt-get update \

    && apt-get install -y curl \ && curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/ script.deb.sh | bash \ && apt-get install -y unzip automake varnish=5.2.0-1~stretch varnish- dev=5.2.0-1~stretch build-essential libtool docutils-common libmhash-dev --allow- unauthenticated \ && curl -O https://download.varnish-software.com/varnish-modules/varnish- modules-0.12.1.tar.gz \ && tar xvzf varnish-modules-0.12.1.tar.gz \ && (cd varnish-modules-0.12.1 && ./configure && make && make install) \ && curl -O -L https://github.com/gquintard/libvmod-accept/archive/master.zip \ && unzip master.zip \ && (cd libvmod-accept-master && ./autogen.sh && ./configure && make && make install) \ && curl -O -L https://github.com/varnish/libvmod-digest/archive/master.zip \ && unzip master.zip \ && (cd libvmod-digest-master && ./autogen.sh && ./configure && make && make install) FROM debian:stretch WORKDIR /etc/varnish EXPOSE 6080 EXPOSE 6081 EXPOSE 6082 EXPOSE 6083 ENV SIZE 100m ENV SECRET E29F6DE6-3803-449E-8A01-AA537A8715B1 ARG BACKEND=nginx
  20. FROM debian:stretch as builder WORKDIR /usr/lib/varnish/vmods/ RUN apt-get update \

    && apt-get install -y curl \ && curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/ script.deb.sh | bash \ && apt-get install -y unzip automake varnish=5.2.0-1~stretch varnish- dev=5.2.0-1~stretch build-essential libtool docutils-common libmhash-dev --allow- unauthenticated \ && curl -O https://download.varnish-software.com/varnish-modules/varnish- modules-0.12.1.tar.gz \ && tar xvzf varnish-modules-0.12.1.tar.gz \ && (cd varnish-modules-0.12.1 && ./configure && make && make install) \ && curl -O -L https://github.com/gquintard/libvmod-accept/archive/master.zip \ && unzip master.zip \ && (cd libvmod-accept-master && ./autogen.sh && ./configure && make && make install) \ && curl -O -L https://github.com/varnish/libvmod-digest/archive/master.zip \ && unzip master.zip \ && (cd libvmod-digest-master && ./autogen.sh && ./configure && make && make install) FROM debian:stretch WORKDIR /etc/varnish EXPOSE 6080 EXPOSE 6081 EXPOSE 6082 EXPOSE 6083 ENV SIZE 100m ENV SECRET E29F6DE6-3803-449E-8A01-AA537A8715B1 ARG BACKEND=nginx
  21. FROM debian:stretch as builder WORKDIR /usr/lib/varnish/vmods/ RUN apt-get update \

    && apt-get install -y curl \ && curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/ script.deb.sh | bash \ && apt-get install -y unzip automake varnish=5.2.0-1~stretch varnish- dev=5.2.0-1~stretch build-essential libtool docutils-common libmhash-dev --allow- unauthenticated \ && curl -O https://download.varnish-software.com/varnish-modules/varnish- modules-0.12.1.tar.gz \ && tar xvzf varnish-modules-0.12.1.tar.gz \ && (cd varnish-modules-0.12.1 && ./configure && make && make install) \ && curl -O -L https://github.com/gquintard/libvmod-accept/archive/master.zip \ && unzip master.zip \ && (cd libvmod-accept-master && ./autogen.sh && ./configure && make && make install) \ && curl -O -L https://github.com/varnish/libvmod-digest/archive/master.zip \ && unzip master.zip \ && (cd libvmod-digest-master && ./autogen.sh && ./configure && make && make install) FROM debian:stretch WORKDIR /etc/varnish EXPOSE 6080 EXPOSE 6081 EXPOSE 6082 EXPOSE 6083 ENV SIZE 100m ENV SECRET E29F6DE6-3803-449E-8A01-AA537A8715B1 ARG BACKEND=nginx
  22. install) \ && curl -O -L https://github.com/varnish/libvmod-digest/archive/master.zip \ && unzip

    master.zip \ && (cd libvmod-digest-master && ./autogen.sh && ./configure && make && make install) FROM debian:stretch WORKDIR /etc/varnish EXPOSE 6080 EXPOSE 6081 EXPOSE 6082 EXPOSE 6083 ENV SIZE 100m ENV SECRET E29F6DE6-3803-449E-8A01-AA537A8715B1 ARG BACKEND=nginx RUN apt-get update \ && apt-get install -y curl \ && curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/ script.deb.sh | bash \ && apt-get install -y varnish=5.2.0-1~stretch libmhash-dev --no-install-recommends --allow-unauthenticated \ && rm -rf /var/lib/apt/lists/* \ && /bin/bash -c ' echo "$SECRET" > /etc/varnish/secret' \ && rm -rf /usr/lib/varnish/vmods/libvmod_*.so COPY --from=builder /usr/lib/varnish/vmods/libvmod_*.so /usr/lib/varnish/vmods/ CMD ["/bin/bash","-c","/usr/sbin/varnishd -j unix,user=varnish -s malloc,$SIZE -F -a internal=0.0.0.0:6080 -a main=0.0.0.0:6081 -a proxy=0.0.0.0:6083,PROXY -T 0.0.0.0:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -p feature=+esi_disable_xml_check - p vsl_mask=+Hash"]
  23. install) \ && curl -O -L https://github.com/varnish/libvmod-digest/archive/master.zip \ && unzip

    master.zip \ && (cd libvmod-digest-master && ./autogen.sh && ./configure && make && make install) FROM debian:stretch WORKDIR /etc/varnish EXPOSE 6080 EXPOSE 6081 EXPOSE 6082 EXPOSE 6083 ENV SIZE 100m ENV SECRET E29F6DE6-3803-449E-8A01-AA537A8715B1 ARG BACKEND=nginx RUN apt-get update \ && apt-get install -y curl \ && curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/ script.deb.sh | bash \ && apt-get install -y varnish=5.2.0-1~stretch libmhash-dev --no-install-recommends --allow-unauthenticated \ && rm -rf /var/lib/apt/lists/* \ && /bin/bash -c ' echo "$SECRET" > /etc/varnish/secret' \ && rm -rf /usr/lib/varnish/vmods/libvmod_*.so COPY --from=builder /usr/lib/varnish/vmods/libvmod_*.so /usr/lib/varnish/vmods/ CMD ["/bin/bash","-c","/usr/sbin/varnishd -j unix,user=varnish -s malloc,$SIZE -F -a internal=0.0.0.0:6080 -a main=0.0.0.0:6081 -a proxy=0.0.0.0:6083,PROXY -T 0.0.0.0:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -p feature=+esi_disable_xml_check - p vsl_mask=+Hash"]
  24. install) \ && curl -O -L https://github.com/varnish/libvmod-digest/archive/master.zip \ && unzip

    master.zip \ && (cd libvmod-digest-master && ./autogen.sh && ./configure && make && make install) FROM debian:stretch WORKDIR /etc/varnish EXPOSE 6080 EXPOSE 6081 EXPOSE 6082 EXPOSE 6083 ENV SIZE 100m ENV SECRET E29F6DE6-3803-449E-8A01-AA537A8715B1 ARG BACKEND=nginx RUN apt-get update \ && apt-get install -y curl \ && curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/ script.deb.sh | bash \ && apt-get install -y varnish=5.2.0-1~stretch libmhash-dev --no-install-recommends --allow-unauthenticated \ && rm -rf /var/lib/apt/lists/* \ && /bin/bash -c ' echo "$SECRET" > /etc/varnish/secret' \ && rm -rf /usr/lib/varnish/vmods/libvmod_*.so COPY --from=builder /usr/lib/varnish/vmods/libvmod_*.so /usr/lib/varnish/vmods/ CMD ["/bin/bash","-c","/usr/sbin/varnishd -j unix,user=varnish -s malloc,$SIZE -F -a internal=0.0.0.0:6080 -a main=0.0.0.0:6081 -a proxy=0.0.0.0:6083,PROXY -T 0.0.0.0:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -p feature=+esi_disable_xml_check - p vsl_mask=+Hash"]
  25. DAEMON_OPTS="-j unix,user=www-data \ -a :80 \ -a :81,PROXY \ -T

    localhost:6082 \ -f /etc/varnish/default.vcl \ -S /etc/varnish/secret \ -l 100m,10m \ -t 60 \ -p feature=+esi_disable_xml_check \ -p connect_timeout=20s \ -p first_byte_timeout=100s \ -p between_bytes_timeout=5s \ -s malloc,3g"
  26. ✓ GET ✓ HEAD - POST - PUT - DELETE

    - PATCH Idempotence Result changes Result doesn't change Don't cache
  27. ✓ Request header ✓ Sent by client (~browser) ✓ Sent

    on every request ✓ Describes state ✓ Is not cached ✓ 3rd party tracking cookies vs own cookies About cookies Cookie: key=value;key2=value2
  28. ✓ Response header ✓ Sent by backend server ✓ Only

    when backend is called ✓ Changes state ✓ Is not cached ✓ Blacklisted for 120s (by default) About cookies Set-Cookie: key=another_value
  29. ✓ 120s by default ✓ Respects HTTP cache-control header ✓

    Respects expires header ✓ Override in VCL file Time to live
  30. Use TTL not to cache Cache-Control: max-age=0 Cache-Control: s-maxage=0 Cache-Control:

    private Cache-Control: no-cache Cache-Control: no-store Expires: Fri, 1 Jan 1971 00:00:00 GMT
  31. Conditional requests HTTP/1.1 200 OK Host: localhost Etag: 7c9d70604c6061da9bb9377d3f00eb27 Content-type:

    text/html; charset=UTF-8 Hello world output GET /if_none_match.php HTTP/1.1 Host: localhost User-Agent: curl/7.48.0
  32. Conditional requests HTTP/1.0 304 Not Modified Host: localhost Etag: 7c9d70604c6061da9bb9377d3f00eb27

    GET /if_none_match.php HTTP/1.1 Host: localhost User-Agent: curl/7.48.0 If-None-Match: 7c9d70604c6061da9bb9377d3f00eb27
  33. Conditional requests HTTP/1.1 200 OK Host: localhost Last-Modified: Fri, 22

    Jul 2016 10:11:16 GMT Content-type: text/html; charset=UTF-8 Hello world output GET /if_none_match.php HTTP/1.1 Host: localhost User-Agent: curl/7.48.0
  34. Conditional requests HTTP/1.0 304 Not Modified Host: localhost Last-Modified: Fri,

    22 Jul 2016 10:11:16 GMT GET /if_none_match.php HTTP/1.1 Host: localhost User-Agent: curl/7.48.0 If-Last-Modified: Fri, 22 Jul 2016 10:11:16 GMT
  35. ✓ URL ✓ Hostname ✓ IP if hostname is not

    set ✓ Vary header Basic variations
  36. VCL Recv VCL Hash VCL Miss VCL Hit VCL Backend

    Response VCL Deliver VCL Purge VCL Error VCL Pipe VCL Pass VCL Synth VCL Backend Error
  37. ✓vcl_recv: receive request ✓vcl_hash: compose cache key ✓vcl_miss: not found

    in cache ✓vcl_hit: found in cache ✓vcl_pass: don’t store in cache ✓vcl_pipe: bypass cache ✓vcl_backend_fetch: connect to backend ✓vcl_backend_response: response from backend ✓vcl_backend_error: backend fetch failed
  38. ✓vcl_purge: after successful purge ✓vcl_synth: send synthetic output ✓vcl_deliver: return

    data to client ✓vcl_init: initialize VMODs ✓vcl_fini: discard VMODs ✓vcl_fail: stop execution
  39. ✓ hash: lookup in cache ✓ pass: don't cache ✓

    synth: synthetic HTML output ✓ pipe: bypass cache ✓ purge: remove from cache VCL_RECV
  40. ✓ fetch: fetch data from backend, don't cache ✓ restart:

    restart transaction ✓ synth: synthetic HTML output VCL_PASS
  41. ✓ deliver: send cached object ✓ miss: synchronous refresh despite

    hit ✓ pass: fetch data from backend despite hit, don't cache ✓ restart: restart transaction ✓ synth: synthetic HTML output VCL_HIT
  42. ✓ fetch: fetch data from backend ✓ pass: fetch data

    from backend, don't cache ✓ restart: restart transaction ✓ synth: synthetic HTML output VCL_MISS
  43. ✓ deliver: deliver object to client ✓ restart: restart transaction

    ✓ synth: synthetic HTML output VCL_DELIVER
  44. ✓ fetch: fetch object from backend ✓ abandon: abandon request

    and send HTTP 503 error VCL_BACKEND_FETCH
  45. ✓ deliver: send fetched data to client ✓ abandon: abandon

    request and send HTTP 503 error ✓ retry: retry backend request VCL_BACKEND_RESPONSE
  46. ✓ vcl_recv: hash ✓ vcl_hash: lookup ✓ vcl_miss: fetch ✓

    vcl_backend_request: fetch ✓ vcl_backend_response: deliver ✓ vcl_deliver: deliver MISS
  47. ✓ vcl_recv: pass ✓ vcl_pass: fetch ✓ vcl_backend_request: fetch ✓

    vcl_backend_response: deliver ✓ vcl_deliver: deliver PASS
  48. ✓ req: incoming request object ✓ req_top: top level esi

    request ✓ bereq: request object to send to backend ✓ beresp: backend response ✓ resp: response to send back to client ✓ obj: cached object ✓ client: client information ✓ server: server information ✓ local: local TCP information ✓ remote: remote TCP information ✓ storage: storage information VCL Objects
  49. ✓ beresp.age ✓ beresp.backend.ip ✓ beresp.backend.name ✓ beresp.do_esi ✓ beresp.grace

    ✓ beresp.keep ✓ beresp.http.set-cookie ✓ beresp.ttl ✓ beresp.status ✓ beresp.uncacheable beresp variables
  50. vcl 4.0; sub vcl_recv { if (req.method == "PRI") {

    return (synth(405)); } if (req.method != "GET" && req.method != "HEAD" && req.method != "PUT" && req.method != "POST" && req.method != "TRACE" && req.method != "OPTIONS" && req.method != “PATCH" && req.method != "DELETE") { return (pipe); } if (req.method != "GET" && req.method != "HEAD") { return (pass); } if (req.http.Authorization || req.http.Cookie) { return (pass); } return (hash); } Idempotence State Action Receive request
  51. sub vcl_hash { hash_data(req.url); if (req.http.host) { hash_data(req.http.host); } else

    { hash_data(server.ip); } return (lookup); } Lookup in cache Variations Action
  52. sub vcl_purge { return (synth(200, "Purged")); } sub vcl_hit {

    if (obj.ttl >= 0s) { return (deliver); } if (obj.ttl + obj.grace > 0s) { return (deliver); } return (miss); } sub vcl_miss { return (fetch); } sub vcl_deliver { return (deliver); } Remove from cache Found in cache Not found in cache Return HTTP response to client
  53. sub vcl_synth { set resp.http.Content-Type = "text/html; charset=utf-8"; set resp.http.Retry-After

    = "5"; synthetic( {"<!DOCTYPE html> <html> <head> <title>"} + resp.status + " " + resp.reason + {"</title> </head> <body> <h1>Error "} + resp.status + " " + resp.reason + {"</h1> <p>"} + resp.reason + {"</p> <h3>Guru Meditation:</h3> <p>XID: "} + req.xid + {"</p> <hr> <p>Varnish cache server</p> </body> </html> "} ); return (deliver); } Send custom HTML
  54. sub vcl_backend_fetch { return (fetch); } sub vcl_backend_response { if

    (bereq.uncacheable) { return (deliver); } else if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Surrogate-control ~ "no-store" || (!beresp.http.Surrogate-Control && beresp.http.Cache-Control ~ "no-cache|no-store|private") || beresp.http.Vary == "*") { set beresp.ttl = 120s; set beresp.uncacheable = true; } return (deliver); } Send backend request Receive backend response TTL State Receive backend response
  55. sub vcl_backend_error { set beresp.http.Content-Type = "text/html; charset=utf-8"; set beresp.http.Retry-After

    = "5"; synthetic( {"<!DOCTYPE html> <html> <head> <title>"} + beresp.status + " " + beresp.reason + {"</title> </head> <body> <h1>Error "} + beresp.status + " " + beresp.reason + {"</h1> <p>"} + beresp.reason + {"</p> <h3>Guru Meditation:</h3> <p>XID: "} + bereq.xid + {"</p> <hr> <p>Varnish cache server</p> </body> </html> "} ); return (deliver); } Send custom HTML on backend error
  56. vcl 4.0; backend default { .host = "127.0.0.1"; .port =

    "80"; .max_connections = 300; .first_byte_timeout = 300s; .connect_timeout = 5s; .between_bytes_timeout = 2s; } More backend work
  57. vcl 4.0; backend default { .host = "127.0.0.1"; .port =

    "80"; .max_connections = 300; .first_byte_timeout = 300s; .connect_timeout = 5s; .between_bytes_timeout = 2s; .probe = { .url = "/"; .interval = 5s; .timeout = 1s; .window = 5; .threshold = 3; } } Even more backend work
  58. vcl 4.0; backend default { .host = "127.0.0.1"; .port =

    "80"; .max_connections = 300; .first_byte_timeout = 300s; .connect_timeout = 5s; .between_bytes_timeout = 2s; .probe = { .request = "HEAD / HTTP/1.1" "Host: localhost" "Connection: close" "User-Agent: Varnish Health Probe"; .interval = 5s; .timeout = 1s; .window = 5; .threshold = 3; } } Even more backend work
  59. vcl 4.0; import directors; import std; import cookie; sub vcl_init

    { new vdir = directors.round_robin(); vdir.add_backend(…); vdir.add_backend(…); } VMOD
  60. vcl 4.0; import std; backend default { .host = "nginx";

    .port = "80"; .probe = { .url = "/"; .interval = 1s; .timeout = 1s; .window = 5; .threshold = 3; } } sub vcl_recv { if(std.healthy(req.backend_hint)) { return(synth(200,"Backend is healthy")); } else { return(synth(200,"Backend is unhealthy")); } } std.healthy
  61. sub vcl_recv { std.timestamp("Before std.log"); std.log("This should appear in the

    Shared Memory Log"); std.timestamp("After std.log"); std.syslog(9,"This should appear in the Syslog"); std.timestamp("After std.syslog"); return(synth(200,"OK")); } Logging
  62. sub vcl_recv { return(synth(200,"Client port: " + std.port(client.ip) + ",

    server port: " + std.port(server.ip))); } std.port
  63. sub vcl_recv { return(synth(200,"Unsorted: " + regsub(req.url,"\?(.+)$","\1") + ", sorted:

    " + regsub(std.querysort(req.url),"\?(.+)$","\1"))); } std.querysort
  64. vcl 4.0; import directors; backend server1 { .host = “1.2.3.4”;

    .port = "80"; .max_connections = 300; .first_byte_timeout = 300s; .connect_timeout = 5s; .between_bytes_timeout = 2s; .probe = { .request = "HEAD / HTTP/1.1" "Host: localhost" "Connection: close" "User-Agent: Varnish Health Probe"; .interval = 5s; .timeout = 1s; .window = 5; .threshold = 3; } } Loadbalancing: server 1
  65. backend server2 { .host = “1.2.3.5”; .port = "80"; .max_connections

    = 300; .first_byte_timeout = 300s; .connect_timeout = 5s; .between_bytes_timeout = 2s; .probe = { .request = "HEAD / HTTP/1.1" "Host: localhost" "Connection: close" "User-Agent: Varnish Health Probe"; .interval = 5s; .timeout = 1s; .window = 5; .threshold = 3; } } Loadbalancing: server 2
  66. sub vcl_init { new vdir = directors.round_robin(); vdir.add_backend(server1); vdir.add_backend(server2); }

    sub vcl_recv { set req.backend_hint = vdir.backend(); } Loadbalancing: round robin
  67. sub vcl_init { new vdir = directors.random(); vdir.add_backend(server1,2); vdir.add_backend(server2,3); }

    sub vcl_recv { set req.backend_hint = vdir.backend(); } Loadbalancing: random
  68. sub vcl_init { new vdir = directors.fallback(); vdir.add_backend(server1); vdir.add_backend(server2); }

    sub vcl_recv { set req.backend_hint = vdir.backend(); } Loadbalancing: fallback
  69. sub vcl_init { new vdir = directors.hash(); vdir.add_backend(server1); vdir.add_backend(server2); }

    sub vcl_recv { set req.backend_hint = vdir.backend(req.url); } Loadbalancing: URL hash
  70. sub vcl_init { new vdir = directors.hash(); vdir.add_backend(server1); vdir.add_backend(server2); }

    sub vcl_recv { set req.backend_hint = vdir.backend(client.identity); } Loadbalancing: IP hash
  71. sub vcl_recv { if(req.url ~ “^/products”) { set req.backend_hint =

    server1; } else { set req.backend_hint = server2; } } Conditional loadbalancing
  72. apt-get install -y varnish varnish-dev build-essential curl -O https://download.varnish-software.com/varnish- modules/varnish-modules-0.12.1.tar.gz

    tar xvzf varnish-modules-0.12.1.tar.gz cd varnish-modules-0.12.1 ./configure make make install Install official VMODs
  73. vcl 4.0; sub vcl_recv { if (req.http.Cookie) { set req.http.Cookie

    = ";" + req.http.Cookie; set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); set req.http.Cookie = regsuball(req.http.Cookie, ";(PHPSESSID)=", "; \1="); set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", ""); if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; } } } Without vmod_cookie
  74. vcl 4.0; import cookie; sub vcl_recv { if (req.http.cookie) {

    cookie.parse(req.http.cookie); cookie.filter_except("PHPSESSID"); set req.http.cookie = cookie.get_string(); if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; } } } With vmod_cookie
  75. vcl 4.0; import cookie; sub vcl_recv { cookie.parse(req.http.cookie); if(cookie.isset("PHPSESSID")) {

    return(synth(200,"PHPSESSID is set and contains the following value: <em>" + cookie.get("PHPSESSID") + "</em>")); } } Other vmod_cookie examples
  76. vcl 4.0; import cookie; sub vcl_recv { cookie.parse(req.http.cookie); if(cookie.isset(“PHPSESSID") &&

    req.url ~ "^/products/[0-9]+$") { cookie.delete("PHPSESSID"); } set req.http.cookie = cookie.get_string(); } Other vmod_cookie examples
  77. vcl 4.0; import accept; backend default { .host = "nginx";

    .port = "80"; } sub vcl_init { new rule = accept.rule("en"); rule.add("nl"); } sub vcl_recv { set req.http.x-old-accept-language= req.http.Accept-Language; set req.http.Accept-Language = rule.filter(req.http.Accept-Language); return(synth(200,"<ul><li>Raw Accept-Language: "+req.http.x-old-accept- language+"</li><li>Filtered Accept-Language: "+req.http.Accept-Language+"</li></ ul>")); } vmod_accept
  78. vcl 4.0; import accept; backend default { .host = "nginx";

    .port = "80"; } sub vcl_init { new language = accept.rule("en"); language.add("nl"); new encoding = accept.rule("deflate"); language.add("gzip"); new contenttype = accept.rule("text/plain"); contenttype.add("text/html"); contenttype.add("application/json"); } sub vcl_recv { set req.http.Accept-Language = language.filter(req.http.Accept-Language); set req.http.Accept-Encoding = encoding.filter(req.http.Accept-Encoding); set req.http.Accept = contenttype.filter(req.http.Accept); } vmod_accept
  79. Accept-Language: nl,en-US;q=0.8,en;q=0.6 Too many variations Impacts hit rate Why is

    vmod_accept useful? Accept-Language: nl Less variations
  80. vcl 4.0; import accept; import std; backend default { .host

    = "nginx"; .port = "80"; } sub vcl_init { new language = accept.rule("en"); language.add("nl"); new encoding = accept.rule(""); language.add("deflate"); language.add("gzip"); new contenttype = accept.rule("text/plain"); contenttype.add("text/html"); contenttype.add("application/json"); } sub vcl_recv { set req.http.Accept-Language = language.filter(req.http.Accept-Language); set req.http.Accept-Encoding = encoding.filter(req.http.Accept-Encoding); set req.http.Accept = contenttype.filter(req.http.Accept); if (req.http.User-Agent ~ "MSIE 6" || req.http.Accept-Encoding ~ "^\s*$") { unset req.http.Accept-Encoding; } set req.http.Host = regsub(req.http.Host, ":[0-9]+", ""); set req.url = std.querysort(req.url); Normalize
  81. if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") { set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|

    siteurl)=([A-z0-9_\-\.%25]+)", ""); set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof| siteurl)=([A-z0-9_\-\.%25]+)", "?"); set req.url = regsub(req.url, "\?&", "?"); set req.url = regsub(req.url, "\?$", ""); } if (req.url ~ "\#") { set req.url = regsub(req.url, "\#.*$", ""); } if (req.url ~ "\?$") { set req.url = regsub(req.url, "\?$", ""); } } Normalize
  82. vcl 4.0; sub vcl_recv { if (req.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv| gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|

    ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz| wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") { unset req.http.Cookie; return (hash); } } sub vcl_backend_response { if (bereq.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac| flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf| ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt| txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") { unset beresp.http.set-cookie; } if (bereq.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4| mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") { unset beresp.http.set-cookie; set beresp.do_stream = true; set beresp.do_gzip = false; } } Cache static assets
  83. vcl 4.0; import std; sub vcl_recv { if (req.url ~

    "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv| gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg| ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz| wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") { unset req.http.Cookie; return (pass); } } Don’t cache static assets
  84. sub vcl_recv { if (req.url ~ "^/status\.php$" || req.url ~

    "^/update\.php$" || req.url ~ "^/admin$" || req.url ~ "^/admin/.*$" || req.url ~ "^/user$" || req.url ~ "^/user/.*$" || req.url ~ "^/flag/.*$" || req.url ~ "^.*/ajax/.*$" || req.url ~ "^.*/ahah/.*$") { return (pass); } } URL blacklist
  85. vcl 4.0; sub vcl_recv { set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(;

    )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", ""); if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; } } Remove tracking cookies
  86. vcl 4.0; sub vcl_recv { if (req.http.Cookie) { set req.http.Cookie

    = ";" + req.http.Cookie; set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); set req.http.Cookie = regsuball(req.http.Cookie, ";(PHPSESSID)=", "; \1="); set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", ""); if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; } } } Only keep session cookie
  87. vcl 4.0; sub vcl_recv { if (req.http.cookie) { cookie.parse(req.http.cookie); cookie.filter_except("PHPSESSID");

    set req.http.cookie = cookie.get_string(); if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; } } } The VMOD way
  88. sub vcl_recv { if (req.http.Cookie) { set req.http.Cookie = ";"

    + req.http.Cookie; set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); set req.http.Cookie = regsuball(req.http.Cookie, ";(language)=", "; \1="); set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", ""); if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; return(pass); } return(hash); } } sub vcl_hash { hash_data(regsub( req.http.Cookie, "^.*language=([^;]*);*.*$", "\1" )); } Language cookie cache variation
  89. vcl 4.0; import cookie; backend default { .host = "nginx";

    .port = "80"; } sub vcl_recv { if (req.http.cookie) { cookie.parse(req.http.cookie); cookie.filter_except("language"); set req.http.cookie = cookie.get_string(); if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; } return(hash); } } sub vcl_hash { hash_data(cookie.get("language")); } The VMOD way
  90. <esi:include src="/header" /> Edge Side Includes ✓Placeholder ✓Parsed by Varnish

    ✓Output is a composition of blocks ✓State per block ✓TTL per block
  91. sub vcl_recv { set req.http.Surrogate-Capability = "key=ESI/1.0"; } sub vcl_backend_response

    { if (beresp.http.Surrogate-Control ~ "ESI/1.0") { unset beresp.http.Surrogate-Control; set beresp.do_esi = true; } } Edge Side Includes
  92. <?php header("Cache-Control: public,must-revalidate,s-maxage=10"); echo "Date in the ESI tag: ".date('Y-m-d

    H:i:s').'<br />'; <?php header("Cache-Control: no-store"); header(“Surrogate-Control: content='ESI/1.0'"); echo '<esi:include src="/esi.php" />'.PHP_EOL; echo "Date in the main page: ".date('Y-m-d H:i:s').'<br />'; Main page ESI frame: esi.php Cached for 10 seconds Not cached
  93. ESI_xmlerror No ESI processing, first char not '<'. (See feature

    esi_disable_xml_check) Expects HTML/ XML tags
  94. vcl 4.0; backend default { .host = "nginx"; .port =

    "80"; } sub vcl_recv { if (req.esi_level > 0) { set req.http.x-parent-url = req_top.url; } } sub vcl_backend_response { if (beresp.http.Surrogate-Control ~ "ESI/1.0") { unset beresp.http.Surrogate-Control; set beresp.do_esi = true; } } Get parent URL
  95. ✓ Server-side ✓ Standardized ✓ Processed on the “edge”, no

    in the browser ✓ Generally faster Edge-Side Includes - Sequential - One fails, all fail - Limited implementation in Varnish
  96. ✓ Client-side ✓ Common knowledge ✓ Parallel processing ✓ Graceful

    degradation AJAX - Processed by the browser - Extra roundtrips - Somewhat slower
  97. {{ render_esi(controller('AppBundle:News:latest', { 'maxPerPage': 5 })) }} {{ render_esi(url('latest_news', {

    'maxPerPage': 5 })) }} In your Twig templates Silex or Symfony Falls back to internal subrequests on failure Does ESI
  98. {{ render_hinclude(controller('AppBundle:News:latest', { 'maxPerPage': 5 })) }} {{ render_hinclude(url(‘latest_news', {

    'maxPerPage': 5 })) }} In your Twig templates Silex or Symfony Requires hinclude.js Does AJAX
  99. <esi:include src="/header" /> ESI vs HInclude <hx:include src="/header"></hx:include> <meta name="include_timeout"

    content="2" /> <meta name="include_mode" content="async" /> Extra parameters
  100. <?php use Symfony\Component\HttpFoundation\Response; require_once dirname(__DIR__).'/vendor/autoload.php'; $app = new Silex\Application(); $app['debug']

    = true; $app->register(new Silex\Provider\TwigServiceProvider(), [ 'twig.path' => dirname(__DIR__).'/views' ]); $app->register(new Silex\Provider\HttpCacheServiceProvider()); $app->register(new Silex\Provider\HttpFragmentServiceProvider()); $app->get('/', function() use($app) { $response = new Response($app['twig']->render('index.twig')); $response->setSharedMaxAge(10); return $response; }); $app->get('/header', function() use($app) { $response = new Response($app['twig']->render('header.twig')); $response->setPrivate(); return $response; })->bind('header'); $app->get('/footer', function() use($app) { $response = new Response($app['twig']->render('footer.twig')); $response->setSharedMaxAge(3); return $response; })->bind('footer'); $app->run();
  101. <!doctype html> <html> <body> {{ render_esi(url('header')) }} <h1>Main content</h1> <dl

    class="row"> <dt class="col-sm-3">Date and time</dt> <dd class="col-sm-9">{{ "now"|date("m/d/Y H:i:s") }}</dd> </dl> {{ render_hinclude(url('footer')) }} <script src="https://rawgit.com/mnot/hinclude/master/hinclude.js"></ script> </body> </html> index.twig
  102. <hr /> <h1>Footer</h1> <dl class="row"> <dt class="col-sm-3">Date and time</dt> <dd

    class="col-sm-9">{{ "now"|date("m/d/Y H:i:s") }}</dd> </dl> footer.twig
  103. sub vcl_backend_response { if (beresp.ttl <= 0s || beresp.http.Set-Cookie ||

    beresp.http.Vary == "*") { set beresp.ttl = 120s; set beresp.uncacheable = true; return (deliver); } } Control Time To Live
  104. sub vcl_deliver { if (obj.hits > 0) { set resp.http.X-Cache

    = "HIT"; } else { set resp.http.X-Cache = "MISS"; } } Debugging
  105. sub vcl_deliver { unset resp.http.X-Powered-By; unset resp.http.Server; unset resp.http.X-Drupal-Cache; unset

    resp.http.X-Varnish; unset resp.http.Via; unset resp.http.Link; unset resp.http.X-Generator; } Anonymize
  106. acl purgers { "localhost"; "127.0.0.1"; "::1"; } sub vcl_recv {

    if (req.method == "PURGE") { if (client.ip !~ purgers) { return (synth(405, "Method not allowed")); } return (purge); } } Purging
  107. <?php header('Cache-control: no-cache'); require 'guzzle.phar'; $client = new GuzzleHttp\Client(['base_uri' =>

    'http://localhost:6081/']); $response = $client->request('PURGE', '/'); if($response->getStatusCode() == 200) { echo "PURGED".PHP_EOL; } else { echo $response->getBody(); } Purging
  108. acl purge { "localhost"; "127.0.0.1"; "::1"; } sub vcl_recv {

    if (req.method == "PURGE") { if (client.ip !~ purgers) { return (synth(405, "Method not allowed")); } if(req.http.x-purge-regex) { ban("req.http.host == " + req.http.host + " && req.url ~ " + req.http.x-purge-regex); } else { ban("req.http.host == " + req.http.host + " && req.url == " + req.url); } return (synth(200, "Purged")); } } Banning
  109. Banning curl -XPURGE "http://example.com/products" curl -XPURGE -H "x-purge-regex:/products" "http://example.com" Add

    to ban list, remove on next request Remove patterns via “X-PURGE-REGEX”
  110. Ban lurker Object User Varnish Server Sends HTTP response Response

    stored in object Sends BAN to Varnish Ban lurker thread Ban list Reads ban list Removes object ban req.http.host == localhost && req.url ~ /products
  111. Ban lurker ban req.http.host == localhost && req.url ~ /products

    Lurker can’t match request info Lurker can only match what’s in the object Next visitor triggers cache remove
  112. acl purge { "localhost"; "127.0.0.1"; "::1"; } sub vcl_recv {

    if (req.method == "PURGE") { if (client.ip !~ purgers) { return (synth(405, "Method not allowed")); } if(req.http.x-purge-regex) { ban("obj.http.x-host == " + req.http.host + " && obj.http.x-url ~ " + req.http.x-purge-regex); } else { ban("obj.http.host == " + req.http.host + " && obj.http.x-url == " + req.url); } return (synth(200, "Purged")); } } sub vcl_backend_response { set beresp.http.x-url = bereq.url; set beresp.http.x-host = bereq.http.host; } sub vcl_deliver { unset resp.http.x-url; unset resp.http.x-host; } Lurker-friendly bans Store request info in response object
  113. Lurker-friendly bans ban obj.http.x-host == localhost && obj.http.x-url ~ /products

    Lurker can match response info Ban lurker removes item from cache async
  114. varnish> param.show ban_lurker_age 200 ban_lurker_age Value is: 60.000 [seconds] (default)

    Minimum is: 0.000 The ban lurker will ignore bans until they are this old. When a ban is added, the active traffic will be tested against it as part of object lookup. Because many applications issue bans in bursts, this parameter holds the ban-lurker off until the rush is over. This should be set to the approximate time which a ban-burst takes.
  115. $ telnet varnish 6082 Connected to localhost. Escape character is

    '^]'. 107 59 cohvooigdtqvkpwewhdxkqiwkfkpwsly Authentication required. auth 5a9c5722f31cc3c92f0e4616571624df7bddde2f8e42aaffe795dc80fb8c91dd 200 240 ----------------------------- Varnish Cache CLI 1.0 ----------------------------- Linux,4.9.49-moby,x86_64,-junix,-smalloc,-smalloc,-hcritbit varnish-5.2.0 revision 4c4875cbf Type 'help' for command list. Type 'quit' to close CLI session. ban obj.http.x-host == localhost && obj.http.x-url ~ /products 200 0
  116. <?php $challenge = $argv[1]; $secret = trim(fgets(STDIN)); $pack = $challenge

    . "\x0A" . $secret . "\x0A" . $challenge . "\x0A"; $key = hash('sha256', $pack); echo $key.PHP_EOL; Use secret & process challenge cohvooigdtqvkpwewhdxkqiwkfkpwsly 5a9c5722f31cc3c92f0e4616571624df7bddde2f8e42aaffe795dc80fb8c91dd E29F6DE6-3803-449E-8A01-AA537A8715B1
  117. $ telnet varnish 6082 Connected to localhost. Escape character is

    '^]'. 107 59 cohvooigdtqvkpwewhdxkqiwkfkpwsly Authentication required. auth 5a9c5722f31cc3c92f0e4616571624df7bddde2f8e42aaffe795dc80fb8c91dd 200 240 ----------------------------- Varnish Cache CLI 1.0 ----------------------------- Linux,4.9.49-moby,x86_64,-junix,-smalloc,-smalloc,-hcritbit varnish-5.2.0 revision 4c4875cbf Type 'help' for command list. Type 'quit' to close CLI session. ban obj.http.x-host == localhost && obj.http.x-url ~ /products 200 0
  118. DAEMON_OPTS="-j unix,user=www-data \ -a :80 \ -a :81,PROXY \ -T

    localhost:6082 \ -f /etc/varnish/default.vcl \ -S /etc/varnish/secret \ -l 100m,10m \ -t 60 \ -p feature=+esi_disable_xml_check \ -p connect_timeout=20s \ -p first_byte_timeout=100s \ -p between_bytes_timeout=5s \ -s malloc,3g"
  119. ----------------------------- Varnish Cache CLI 1.0 ----------------------------- Linux,4.9.49-moby,x86_64,-junix,-smalloc,-smalloc,-hcritbit varnish-5.2.0 revision 4c4875cbf

    Type 'help' for command list. Type 'quit' to close CLI session. help 200 613 auth <response> backend.list [-p] [<backend_pattern>] backend.set_health <backend_pattern> [auto|healthy|sick] ban <field> <operator> <arg> [&& <field> <oper> <arg> ...] ban.list banner help [<command>] panic.clear [-z] panic.show param.set <param> <value> param.show [-l] [<param>] ping [<timestamp>] quit start status stop storage.list vcl.discard <configname|label> vcl.inline <configname> <quoted_VCLstring> [auto|cold|warm] vcl.label <label> <configname> vcl.list vcl.load <configname> <filename> [auto|cold|warm] vcl.show [-v] <configname> vcl.state <configname> [auto|cold|warm] vcl.use <configname|label>
  120. vcl 4.0; acl refreshers { "172.18.0.0"/24; "localhost"; } backend default

    { .host = "nginx"; .port = "80"; } sub vcl_recv { if (req.method == "REFRESH") { if (client.ip !~ refreshers) { return (synth(405, "Method not allowed")); } set req.method = "GET"; set req.hash_always_miss = true; } } Refresh content Perform miss, fetch content, overwrite object
  121. vcl 4.0; import xkey; acl purgers { "172.18.0.0"/24; } backend

    default { .host = "nginx"; .port = "80"; } sub vcl_recv { if (req.method == "PURGE") { if (client.ip !~ purgers) { return (synth(405, "Method now allowed")); } set req.http.n-gone = xkey.purge(req.http.key); return (synth(200, "Invalidated "+req.http.n-gone+" objects for key "+ req.http.key)); } if (req.method == "SOFTPURGE") { if (client.ip !~ purgers) { return (synth(405, "Method not allowed")); } set req.http.n-gone = xkey.softpurge(req.http.key); return (synth(200, "Invalidated "+req.http.n-gone+" objects via softpurge for key "+ req.http.key)); } }
  122. Invalidate tags curl -XPURGE -H"key: b" "http://example.com/" PURGE / HTTP/1.1

    Host: localhost User-Agent: curl/7.55.1 Accept: */* key:b
  123. Header-based invalidation If-Modified-Since: Tue, 14 Jun 2016 11:49:18 GMT Last-Modified:

    Tue, 14 Jun 2016 11:49:32 GMT Cache-Control: max-age=10 HTTP 200: OK HTTP 304: Not modified
  124. usage: varnishstat [-1lV] [-f field] [-t seconds|<off>] [-n varnish_name] [-N

    filename] -1 # Print the statistics to stdout. -f field # Field inclusion glob # If it starts with '^' it is used as an exclusion list. -l # Lists the available fields to use with the -f option. -n varnish_name # The varnishd instance to get logs from. -N filename # Filename of a stale VSM instance. -t seconds|<off> # Timeout before returning error on initial VSM connection. -V # Display the version number and exit. -x # Print statistics to stdout as XML. -j # Print statistics to stdout as JSON. Varnishstat usage
  125. ~# varnishstat -f MAIN.cache_hit -1 MAIN.cache_hit 13049135 5.39 Cache hits

    Varnishstat usage ~# varnishstat -f MAIN.cache_hit -j -1 { "timestamp": "2016-06-14T16:10:32", "MAIN.cache_hit": { "description": "Cache hits", "type": "MAIN", "flag": "c", "format": "i", "value": 13050992 } }
  126. Varnishstat usage ~# varnishstat -f MAIN.n_object -f MAIN.n_lru_nuked -j {

    "timestamp": "2016-06-14T16:14:49", "MAIN.n_object": { "description": "object structs made", "type": "MAIN", "flag": "g", "format": "i", "value": 46295 }, "MAIN.n_lru_nuked": { "description": "Number of LRU nuked objects", "type": "MAIN", "flag": "g", "format": "i", "value": 0 } }
  127. ✓ Session ✓ Client ✓ Uptime ✓ Hit/miss ✓ Backend

    ✓ Fetch ✓ Threading ✓ Cache objects ✓ Memory ✓ Invalidation Varnishstat counters
  128. VSL

  129. ✓ In-memory logs ✓ Generated by varnishd ✓ 81 MB

    by default ✓ Customize with “-l” setting ✓ Varnishlog command ✓ Varnishtop command VSL
  130. * << Request >> 10973258 - Begin req 10973257 rxreq

    - Timestamp Start: 1501507281.942533 0.000000 0.000000 - Timestamp Req: 1501507281.942533 0.000000 0.000000 - ReqStart 127.0.0.1 59753 - ReqMethod GET - ReqURL / - ReqProtocol HTTP/1.1 - ReqHeader Host: feryn.eu
  131. ✓ Items of work ✓ Identified by VXID ✓ 2

    kinds: ✓ Sessions ✓ Requests Transactions
  132. ✓ Identifies TCP connection ✓ Contains multiple requests Transactions ✓

    Client request ✓ Backend request ✓ ESI subrequest Session Request
  133. * << BeReq >> 98318 - Begin bereq 98317 fetch

    - BereqURL / * << BeReq >> 98320 - Begin bereq 98319 fetch - BereqURL /header * << Request >> 98319 - Begin req 98317 esi - ReqURL /header - Link bereq 98320 fetch * << BeReq >> 98322 - Begin bereq 98321 fetch - BereqURL /nav * << Request >> 98321 - Begin req 98317 esi - ReqURL /nav - Link bereq 98322 fetch * << Request >> 98317 - Begin req 98316 rxreq - ReqURL / - Link bereq 98318 fetch - Link req 98319 esi - Link req 98321 esi * << BeReq >> 98324 - Begin bereq 98323 fetch - BereqURL /footer * << Request >> 98323 - Begin req 98316 rxreq - ReqURL /footer - Link bereq 98324 fetch * << Session >> 98316 - Begin sess 0 HTTP/1 - Link req 98317 rxreq - Link req 98323 rxreq
  134. * << Session >> 14 - Begin sess 0 HTTP/1

    - Link req 65539 rxreq - Link req 65545 rxreq ** << Request >> 65539 -- Begin req 14 rxreq -- ReqURL / -- Link bereq 65540 fetch -- Link req 65541 esi -- Link req 65543 esi ** << Request >> 65545 -- Begin req 14 rxreq -- ReqURL /footer -- Link bereq 65546 fetch *** << BeReq >> 65540 --- Begin bereq 65539 fetch --- BereqURL / *** << Request >> 65541 --- Begin req 65539 esi --- ReqURL /header --- Link bereq 65542 fetch *** << Request >> 65543 --- Begin req 65539 esi --- ReqURL /nav --- Link bereq 65544 fetch *** << BeReq >> 65546 --- Begin bereq 65545 fetch --- BereqURL /footer **** << BeReq >> 65542 ---- Begin bereq 65541 fetch ---- BereqURL /header **** << BeReq >> 65544 ---- Begin bereq 65543 fetch ---- BereqURL /nav
  135. - ReqStart 127.0.0.1 56312 - ReqMethod GET - ReqURL /

    - ReqProtocol HTTP/1.1 - ReqHeader Host: localhost - ReqHeader Connection: keep-alive - ReqHeader Upgrade-Insecure-Requests: 1 - ReqHeader User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ 59.0.3071.115 Safari/537.36 - ReqHeader Accept: text/html,application/xhtml+xml,application/ xml;q=0.9,image/webp,image/apng,*/*;q=0.8 - ReqHeader Accept-Encoding: gzip, deflate, br - ReqHeader Accept-Language: nl,en-US;q=0.8,en;q=0.6 - ReqHeader X-Forwarded-For: 127.0.0.1
  136. - RespProtocol HTTP/1.1 - RespStatus 200 - RespReason OK -

    RespHeader Host: localhost - RespHeader Cache-Control: public, s-maxage=500 - RespHeader Date: Tue, 01 Aug 2017 08:56:44 GMT - RespHeader ETag: "c5afddc587599a72d467caca23e980bf" - RespHeader Vary: Accept-Language - RespHeader Content-Length: 3098 - RespHeader Content-Type: text/html; charset=UTF-8 - RespHeader X-Varnish: 32770 - RespHeader Age: 10
  137. * << Request >> 5 - ReqURL / - VCL_call

    RECV - VCL_return hash - VCL_call HASH - VCL_return lookup - VCL_call MISS - VCL_return fetch - VCL_call DELIVER - VCL_return deliver ** << BeReq >> 6 -- BereqURL / -- VCL_call BACKEND_FETCH -- VCL_return fetch -- VCL_call BACKEND_RESPONSE -- VCL_return deliver
  138. * << Request >> 6 - ReqURL / - VCL_call

    RECV - ReqURL / - VCL_return hash - VCL_call HASH - VCL_return lookup - VCL_call HIT - VCL_return deliver - VCL_call DELIVER - VCL_return deliver
  139. * << Request >> 32789 - ReqURL /header - ExpBan

    98355 banned lookup ExpBan tag
  140. * << Request >> 98369 - ReqURL /footer - Hit

    65597 Hit tag varnishlog -i "ReqUrl,Hit"
  141. %s %d %d %d %d [ %d %d %u %u

    ] | | | | | | | | | | | | | | | | | +- Max-Age from Cache-Control header | | | | | | | +---- Expires header | | | | | | +------- Date header | | | | | +---------- Age (incl Age: header value) | | | | +--------------- Reference time for TTL | | | +------------------ Keep | | +--------------------- Grace | +------------------------ TTL +--------------------------- "RFC" or "VCL" TTL tag
  142. -- TTL VCL 120 10 0 1501597242 ✓ TTL decided

    by the VCL ✓ 120 seconds cached ✓ 10 seconds grace time ✓ 0 seconds keep time ✓ Reference time: 1501597242
 (2017-08-01 14:20:42)
  143. -- RFC 500 10 -1 1501598872 1501598872 1501598872 0 500

    ✓ 500 seconds TTL (via headers) ✓ 10 seconds grace ✓ No keep value ✓ 2017-08-01 14:47:52 date, age & reference time ✓ Cache-control headers sets 500 second TTL
  144. -- RFC 500 10 -1 1501598872 1501598869 1501598872 0 500

    ✓ Don’t screw with the age header ✓ Only 497 second effective TTL ✓ Custom Age header (3 seconds off)
  145. - Begin bereq 98317 fetch - Begin req 98317 esi

    - Begin req 98316 rxreq - Begin sess 0 HTTP/1
  146. %s: %f %f %f | | | | | |

    | +- Time since last timestamp | | +---- Time since start of work unit | +------- Absolute time of event +----------- Event label Timestamp tag
  147. * << Request >> 65539 - Timestamp Start: 1501601912.758662 0.000000

    0.000000 - Timestamp Req: 1501601912.758662 0.000000 0.000000 - Timestamp Fetch: 1501601912.806733 0.048071 0.048071 - Timestamp Process: 1501601912.806750 0.048088 0.000017 - Timestamp Resp: 1501601912.806787 0.048125 0.000037 ** << BeReq >> 65540 -- Timestamp Start: 1501601912.758753 0.000000 0.000000 -- Timestamp Bereq: 1501601912.758952 0.000199 0.000199 -- Timestamp Beresp: 1501601912.806677 0.047924 0.047725 -- Timestamp BerespBody: 1501601912.806749 0.047996 0.000072
  148. ✓ -i: include tags ✓ -I: include tags by regex

    ✓ -x: exclude tags ✓ -X: exclude by regex Filtering output
  149. * << Session >> 252394 ** << Request >> 252395

    -- ReqURL / -- VCL_call RECV -- VCL_return hash -- VCL_call HASH -- VCL_return lookup -- VCL_call HIT -- VCL_return deliver -- VCL_call DELIVER -- VCL_return deliver *** << Request >> 252397 --- ReqURL /header --- VCL_call RECV --- VCL_return hash --- VCL_call HASH --- VCL_return lookup --- VCL_call HIT --- VCL_return deliver --- VCL_call DELIVER --- VCL_return deliver *** << Request >> 252399 --- ReqURL /nav --- VCL_call RECV --- VCL_return hash --- VCL_call HASH --- VCL_return lookup --- VCL_call HIT --- VCL_return deliver --- VCL_call DELIVER --- VCL_return deliver *** << BeReq >> 252396 --- VCL_call BACKEND_FETCH --- VCL_return fetch --- VCL_call BACKEND_RESPONSE --- VCL_return deliver **** << BeReq >> 252398 ---- VCL_call BACKEND_FETCH ---- VCL_return fetch ---- VCL_call BACKEND_RESPONSE ---- VCL_return deliver **** << BeReq >> 252400 ---- VCL_call BACKEND_FETCH ---- VCL_return fetch ---- VCL_call BACKEND_RESPONSE ---- VCL_return deliver
  150. * << Request >> 314125 - ReqStart 127.0.0.1 64585 -

    ReqMethod GET - ReqURL / - ReqProtocol HTTP/1.1 - ReqAcct 476 0 476 311 0 311 * << Request >> 314126 - ReqStart 127.0.0.1 64585 - ReqMethod GET - ReqURL /footer - ReqProtocol HTTP/1.1 - ReqAcct 370 0 370 309 0 309
  151. * << Request >> 374378 - ReqURL / - ReqHeader

    Accept-Language: nl,en-US;q=0.8,en;q=0.6 * << Request >> 374379 - ReqURL /footer - ReqHeader Accept-Language: nl,en-US;q=0.8,en;q=0.6
  152. * << Request >> 374384 - ReqURL /footer - RespHeader

    Host: localhost - RespHeader X-Powered-By: PHP/7.0.15 - RespHeader Cache-Control: public, s-maxage=500 - RespHeader Date: Wed, 02 Aug 2017 11:27:21 GMT - RespHeader ETag: "d47ac09f5351f8f4c97c99ef5b3d2ecd" - RespHeader Vary: Accept-Language - RespHeader Content-Length: 80 - RespHeader Content-Type: text/html; charset=UTF-8 - RespHeader X-Varnish: 374384 15367 - RespHeader Age: 334 - RespHeader Via: 1.1 varnish-v4 - RespHeader Connection: keep-alive
  153. * << Request >> 374384 - ReqURL /footer - RespHeader

    Host: localhost - RespHeader Cache-Control: public, s-maxage=500 - RespHeader Date: Wed, 02 Aug 2017 11:27:21 GMT - RespHeader ETag: "d47ac09f5351f8f4c97c99ef5b3d2ecd" - RespHeader Vary: Accept-Language - RespHeader Content-Length: 80 - RespHeader Content-Type: text/html; charset=UTF-8 - RespHeader Age: 334 - RespHeader Via: 1.1 varnish-v4 - RespHeader Connection: keep-alive
  154. * << Request >> 59383 - ReqStart 127.0.0.1 53195 -

    ReqMethod GET - ReqURL / - ReqHeader Host: localhost - ReqHeader Connection: keep-alive - ReqHeader Cache-Control: max-age=0 - ReqHeader Upgrade-Insecure-Requests: 1 - ReqHeader User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36 - ReqHeader Accept: text/ html,application/ xhtml+xml,application/xml;q=0.9,image/ webp,image/apng,*/*;q=0.8 - ReqHeader Accept-Encoding: gzip, deflate, br - ReqHeader Accept-Language: nl,en- US;q=0.8,en;q=0.6 - ReqHeader If-None-Match: W/"27f341f8e459dd35f1087c55351cacda" - ReqHeader X-Forwarded-For: 127.0.0.1 - ReqUnset Accept-Language: nl,en- US;q=0.8,en;q=0.6 - ReqHeader accept-language: nl - ReqHeader Surrogate-Capability: key=ESI/1.0 - ReqUnset Accept-Encoding: gzip, deflate, br - ReqHeader Accept-Encoding: gzip - RespHeader Host: localhost - RespHeader Cache-Control: public, s- maxage=500 - RespHeader Date: Wed, 02 Aug 2017 11:46:49 GMT - RespHeader ETag: "27f341f8e459dd35f1087c55351cacda" - RespHeader Vary: Accept-Language - RespHeader Content-Length: 3098 - RespHeader Content-Type: text/html; charset=UTF-8 - RespHeader Age: 152 - RespHeader Via: 1.1 varnish-v4 - RespHeader ETag: W/"27f341f8e459dd35f1087c55351cacda" - RespHeader Connection: keep-alive - Timestamp Resp: 1501674561.472358 0.000068 0.000020
  155. ✓ All response headers ✓ All tags that start with

    “Req” ✓ Exclude “x-“ response headers ✓ Include “response time” timestamp ✓ Exclude request protocol log lines, and the request accountancy log lines All-in-one
  156. * << Request >> 374400 - VCL_call RECV - VCL_return

    hash - VCL_call HASH - VCL_return lookup - VCL_call HIT - VCL_return deliver - VCL_call DELIVER - VCL_return deliver
  157. varnishlog -i ReqUrl -q "VCL_call eq 'MISS' or VCL_call eq

    'PASS'" varnishlog -i ReqUrl -I "Timestamp:Resp" -q "Timestamp:Resp[2] > 1.0"
  158. varnishlog -n myVarnishInstance varnishlog -d varnishlog -w file varnishlog -r

    file varnishlog -A -a -w file varnishlog -i "ReqUrl,VCL_*" -D -a -A -w /var/log/varnish/custom.log - P /var/run/custom_varnishlog.pid
  159. $ varnishtop -I reqheader:Accept-Language -q "ReqUrl eq '/'" list length

    4 0.86 ReqHeader Accept-Language: en-US 0.80 ReqHeader Accept-Language: nl-NL,nl;q=0.8,en- US;q=0.6,en;q=0.4 0.54 ReqHeader Accept-Language: nl,en-US;q=0.8,en;q=0.6 0.39 ReqHeader Accept-Language: nl-BE
  160. * << Request >> 12 - ReqMethod GET - ReqURL

    / - ReqProtocol HTTP/1.1 - ReqHeader Host: localhost - ReqHeader User-Agent: curl/7.48.0 - ReqHeader Accept: */* - ReqHeader Cookie: myCookie=bla - ReqHeader X-Forwarded-For: 127.0.0.1 - VCL_call RECV - VCL_return pass - VCL_call HASH - VCL_return lookup - VCL_call PASS - VCL_return fetch - VCL_call DELIVER - VCL_return deliver
  161. << Request >> 32779 - ReqMethod POST - ReqURL /

    - ReqProtocol HTTP/1.1 - ReqHeader Host: localhost - ReqHeader User-Agent: curl/7.48.0 - ReqHeader Accept: */* - ReqHeader X-Forwarded-For: 127.0.0.1 - VCL_call RECV - VCL_return pass - VCL_call HASH - VCL_return lookup - VCL_call PASS - VCL_return fetch - VCL_call DELIVER - VCL_return deliver
  162. * << Request >> 15 - ReqMethod GET - ReqURL

    / - ReqProtocol HTTP/1.1 - ReqHeader Host: localhost - ReqHeader Authorization: Basic dGhpanM6ZmVyeW4= - ReqHeader User-Agent: curl/7.48.0 - ReqHeader Accept: */* - ReqHeader X-Forwarded-For: 127.0.0.1 - VCL_call RECV - VCL_return pass - VCL_call HASH - VCL_return lookup - VCL_call PASS - VCL_return fetch - VCL_call DELIVER - VCL_return deliver
  163. * << Request >> 19010384 - ReqURL /my-url - VCL_call

    RECV - VCL_return hash - VCL_call HASH - VCL_return lookup - VCL_call PASS - VCL_return fetch - VCL_call DELIVER - VCL_return deliver ** << BeReq >> 19010385 -- VCL_call BACKEND_FETCH -- VCL_return fetch -- BerespProtocol HTTP/1.1 -- BerespStatus 200 -- BerespReason OK -- BerespHeader Date: Thu, 03 Aug 2017 08:15:22 GMT -- BerespHeader Server: Apache/ 2.4.10 (Debian) -- BerespHeader Last-Modified: Tue, 01 Aug 2017 07:21:00 GMT -- BerespHeader ETag: "5c0d-555abfd3f422f-gzip" -- BerespHeader Vary: Accept- Encoding -- BerespHeader Content-Encoding: gzip -- BerespHeader Cache-Control: max- age=0 -- BerespHeader Expires: Thu, 03 Aug 2017 08:15:22 GMT -- BerespHeader Content-Length: 7686 -- BerespHeader Content-Type: application/json -- TTL RFC 0 10 -1 1501748123 1501748123 1501748122 1501748122 0 -- VCL_call BACKEND_RESPONSE -- TTL VCL 120 10 0 1501748123 -- VCL_return deliver
  164. * << Request >> 65551 - ReqURL /set-cookie - VCL_call

    RECV - VCL_return hash - VCL_call HASH - VCL_return lookup - VCL_call PASS - VCL_return fetch - VCL_call DELIVER - VCL_return deliver ** << BeReq >> 65552 -- VCL_call BACKEND_FETCH -- VCL_return fetch -- BerespProtocol HTTP/1.1 -- BerespStatus 200 -- BerespReason OK -- BerespHeader Cache-control: s-maxage=10 -- BerespHeader Set-Cookie: myCookie=bla -- BerespHeader Content-type: text/html; charset=UTF-8 -- BerespHeader Date: Thu, 03 Aug 2017 08:39:04 GMT -- TTL RFC 10 10 -1 1501749545 1501749545 1501749544 0 10 -- VCL_call BACKEND_RESPONSE -- TTL VCL 120 10 0 1501749545 -- VCL_return deliver
  165. vcl 4.0; sub vcl_recv { if (req.method == "PRI") {

    return (synth(405)); } if (req.method != "GET" && req.method != "HEAD" && req.method != "PUT" && req.method != "POST" && req.method != "TRACE" && req.method != "OPTIONS" && req.method != “PATCH" && req.method != "DELETE") { return (pipe); } if (req.method != "GET" && req.method != "HEAD") { return (pass); } if (req.http.Authorization || req.http.Cookie) { return (pass); } return (hash); }
  166. eyJpc3MiOiJodHRwOlwvXC 9sb2NhbGhvc3RcLyIsImlh dCI6MTUwODc0NzM2MywiZX hwIjoxNTA4NzUwOTYzLCJ1 aWQiOjEsInVzZXJuYW1lIj oiVGhpanMifQ { "alg": "HS256", "typ":

    "JWT" } { "iss": "http://localhost/", "iat": 1508746861, "exp": 1508750461, "uid": 1, "username": "Thijs" } HMACSHA256( base64UrlEncode(header) + "." + base64UrlEncode(payload), secret ) eyJ0eXAiOiJKV1QiLCJhb GciOiJIUzI1NiJ9 EPemqBaH74Sbs0bZqAaR8i 7uVYO389VOlJvWC3ocL7g
  167. User Varnish Server Issues JWT Validates JWT Reads JWT Knows

    secret key Knows secret key Does not know secret key
  168. vcl 4.0; import std; import var; import cookie; import digest;

    backend default { .host = "nginx"; .port = "80"; } sub vcl_recv { if (req.http.Cookie) { cookie.parse(req.http.cookie); cookie.filter_except("jwt_cookie"); set req.http.cookie = cookie.get_string(); if (req.http.Cookie ~ "^\s*$") { unset req.http.Cookie; } } if ((req.method != "GET" && req.method != "HEAD") || req.http.Authorization) { return (pass); } call jwt; return(hash); }
  169. vcl 4.0; import std; import var; import cookie; import digest;

    backend default { .host = "nginx"; .port = "80"; } sub vcl_recv { if (req.http.Cookie) { cookie.parse(req.http.cookie); cookie.filter_except("jwt_cookie"); set req.http.cookie = cookie.get_string(); if (req.http.Cookie ~ "^\s*$") { unset req.http.Cookie; } } if ((req.method != "GET" && req.method != "HEAD") || req.http.Authorization) { return (pass); } call jwt; return(hash); }
  170. vcl 4.0; import std; import var; import cookie; import digest;

    backend default { .host = "nginx"; .port = "80"; } sub vcl_recv { if (req.http.Cookie) { cookie.parse(req.http.cookie); cookie.filter_except("jwt_cookie"); set req.http.cookie = cookie.get_string(); if (req.http.Cookie ~ "^\s*$") { unset req.http.Cookie; } } if ((req.method != "GET" && req.method != "HEAD") || req.http.Authorization) { return (pass); } call jwt; return(hash); }
  171. vcl 4.0; import std; import var; import cookie; import digest;

    backend default { .host = "nginx"; .port = "80"; } sub vcl_recv { if (req.http.Cookie) { cookie.parse(req.http.cookie); cookie.filter_except("jwt_cookie"); set req.http.cookie = cookie.get_string(); if (req.http.Cookie ~ "^\s*$") { unset req.http.Cookie; } } if ((req.method != "GET" && req.method != "HEAD") || req.http.Authorization) { return (pass); } call jwt; return(hash); }
  172. vcl 4.0; import std; import var; import cookie; import digest;

    backend default { .host = "nginx"; .port = "80"; } sub vcl_recv { if (req.http.Cookie) { cookie.parse(req.http.cookie); cookie.filter_except("jwt_cookie"); set req.http.cookie = cookie.get_string(); if (req.http.Cookie ~ "^\s*$") { unset req.http.Cookie; } } if ((req.method != "GET" && req.method != "HEAD") || req.http.Authorization) { return (pass); } call jwt; return(hash); }
  173. sub vcl_synth { #301 & 302 synths should be actual

    redirects if (resp.status == 301 || resp.status == 302) { set resp.http.location = resp.reason; set resp.reason = "Moved"; return (deliver); } }
  174. sub jwt { var.set("key",std.fileread("/etc/varnish/jwt.key")); std.log("Ready to perform some JWT magic");

    if(cookie.isset("jwt_cookie")) { var.set("token", cookie.get("jwt_cookie")); var.set("header", regsub(var.get("token"),"([^\.]+)\.[^\.]+\.[^\.]+","\1")); var.set("type", regsub(digest.base64url_decode(var.get("header")),{"^.*?"typ"\s*: \s*"(\w+)".*?$"},"\1")); var.set("algorithm", regsub(digest.base64url_decode(var.get("header")), {"^.*?"alg"\s*:\s*"(\w+)".*?$"},"\1")); if(var.get("type") == "JWT" && var.get("algorithm") == "HS256") { var.set("rawPayload",regsub(var.get("token"),"[^\.]+\.([^\.]+)\.[^\.]+$","\1")); var.set("signature",regsub(var.get("token"),"^[^\.]+\.[^\.]+\.([^\.]+)$","\1")); var.set("currentSignature",digest.base64url_nopad_hex(digest.hmac_sha256(var.get("key"), var.get("header") + "." + var.get("rawPayload")))); var.set("payload", digest.base64url_decode(var.get("rawPayload"))); var.set("exp",regsub(var.get("payload"),{"^.*?"exp"\s*:\s*([0-9]+).*?$"},"\1")); var.set("userId",regsub(var.get("payload"),{"^.*?"uid"\s*:\s*([0-9]+).*? $"},"\1")); if(var.get("userId") ~ "^\d+$") { if(std.time(var.get("exp"),now) >= now) { if(var.get("signature") == var.get("currentSignature")) { set req.http.X-Login="true"; } else { std.log("JWT: signature doesn't match. Received: " + var.get("signature") + ", expected: " + var.get("currentSignature"));
  175. sub jwt { var.set("key",std.fileread("/etc/varnish/jwt.key")); std.log("Ready to perform some JWT magic");

    if(cookie.isset("jwt_cookie")) { var.set("token", cookie.get("jwt_cookie")); var.set("header", regsub(var.get("token"),"([^\.]+)\.[^\.]+\.[^\.]+","\1")); var.set("type", regsub(digest.base64url_decode(var.get("header")),{"^.*?"typ"\s*: \s*"(\w+)".*?$"},"\1")); var.set("algorithm", regsub(digest.base64url_decode(var.get("header")), {"^.*?"alg"\s*:\s*"(\w+)".*?$"},"\1")); if(var.get("type") == "JWT" && var.get("algorithm") == "HS256") { var.set("rawPayload",regsub(var.get("token"),"[^\.]+\.([^\.]+)\.[^\.]+$","\1")); var.set("signature",regsub(var.get("token"),"^[^\.]+\.[^\.]+\.([^\.]+)$","\1")); var.set("currentSignature",digest.base64url_nopad_hex(digest.hmac_sha256(var.get("key"), var.get("header") + "." + var.get("rawPayload")))); var.set("payload", digest.base64url_decode(var.get("rawPayload"))); var.set("exp",regsub(var.get("payload"),{"^.*?"exp"\s*:\s*([0-9]+).*?$"},"\1")); var.set("userId",regsub(var.get("payload"),{"^.*?"uid"\s*:\s*([0-9]+).*? $"},"\1")); if(var.get("userId") ~ "^\d+$") { if(std.time(var.get("exp"),now) >= now) { if(var.get("signature") == var.get("currentSignature")) { set req.http.X-Login="true"; } else { std.log("JWT: signature doesn't match. Received: " + var.get("signature") + ", expected: " + var.get("currentSignature"));
  176. sub jwt { var.set("key",std.fileread("/etc/varnish/jwt.key")); std.log("Ready to perform some JWT magic");

    if(cookie.isset("jwt_cookie")) { var.set("token", cookie.get("jwt_cookie")); var.set("header", regsub(var.get("token"),"([^\.]+)\.[^\.]+\.[^\.]+","\1")); var.set("type", regsub(digest.base64url_decode(var.get("header")),{"^.*?"typ"\s*: \s*"(\w+)".*?$"},"\1")); var.set("algorithm", regsub(digest.base64url_decode(var.get("header")), {"^.*?"alg"\s*:\s*"(\w+)".*?$"},"\1")); if(var.get("type") == "JWT" && var.get("algorithm") == "HS256") { var.set("rawPayload",regsub(var.get("token"),"[^\.]+\.([^\.]+)\.[^\.]+$","\1")); var.set("signature",regsub(var.get("token"),"^[^\.]+\.[^\.]+\.([^\.]+)$","\1")); var.set("currentSignature",digest.base64url_nopad_hex(digest.hmac_sha256(var.get("key"), var.get("header") + "." + var.get("rawPayload")))); var.set("payload", digest.base64url_decode(var.get("rawPayload"))); var.set("exp",regsub(var.get("payload"),{"^.*?"exp"\s*:\s*([0-9]+).*?$"},"\1")); var.set("userId",regsub(var.get("payload"),{"^.*?"uid"\s*:\s*([0-9]+).*? $"},"\1")); if(var.get("userId") ~ "^\d+$") { if(std.time(var.get("exp"),now) >= now) { if(var.get("signature") == var.get("currentSignature")) { set req.http.X-Login="true"; } else { std.log("JWT: signature doesn't match. Received: " + var.get("signature") + ", expected: " + var.get("currentSignature"));
  177. var.get("header") + "." + var.get("rawPayload")))); var.set("payload", digest.base64url_decode(var.get("rawPayload"))); var.set("exp",regsub(var.get("payload"),{"^.*?"exp"\s*:\s*([0-9]+).*?$"},"\1")); var.set("userId",regsub(var.get("payload"),{"^.*?"uid"\s*:\s*([0-9]+).*? $"},"\1"));

    if(var.get("userId") ~ "^\d+$") { if(std.time(var.get("exp"),now) >= now) { if(var.get("signature") == var.get("currentSignature")) { set req.http.X-Login="true"; } else { std.log("JWT: signature doesn't match. Received: " + var.get("signature") + ", expected: " + var.get("currentSignature")); } } else { std.log("JWT: token has expired"); } } else { std.log("UserId '"+ var.get("userId") +"', is not numeric"); } } else { std.log("JWT: type is not JWT or algorithm is not HS256"); } std.log("JWT processing finished. UserId: " + var.get("userId") + ". X-Login: " + req.http.X-login); } if(req.url ~ "^/(\?[^\?]+)?$" && req.http.X-Login != "true") { return(synth(302,"/post.php")); } }
  178. var.get("header") + "." + var.get("rawPayload")))); var.set("payload", digest.base64url_decode(var.get("rawPayload"))); var.set("exp",regsub(var.get("payload"),{"^.*?"exp"\s*:\s*([0-9]+).*?$"},"\1")); var.set("userId",regsub(var.get("payload"),{"^.*?"uid"\s*:\s*([0-9]+).*? $"},"\1"));

    if(var.get("userId") ~ "^\d+$") { if(std.time(var.get("exp"),now) >= now) { if(var.get("signature") == var.get("currentSignature")) { set req.http.X-Login="true"; } else { std.log("JWT: signature doesn't match. Received: " + var.get("signature") + ", expected: " + var.get("currentSignature")); } } else { std.log("JWT: token has expired"); } } else { std.log("UserId '"+ var.get("userId") +"', is not numeric"); } } else { std.log("JWT: type is not JWT or algorithm is not HS256"); } std.log("JWT processing finished. UserId: " + var.get("userId") + ". X-Login: " + req.http.X-login); } if(req.url ~ "^/(\?[^\?]+)?$" && req.http.X-Login != "true") { return(synth(302,"/post.php")); } }
  179. Vary: X-Login Vary header sent by application X-Login header set

    by Varnish Creates cache variations in Varnish
  180. ✓ Cache-control ✓ Content negotiation ✓ Cache variations ✓ Conditional

    requests ✓ Content composition ✓ JWT authentication Checklist