Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Better Metrics through Scripting

Matt Williams
September 06, 2017

Better Metrics through Scripting

NGINX Conf 2017

Matt Williams

September 06, 2017
Tweet

More Decks by Matt Williams

Other Decks in Technology

Transcript

  1. Better Metrics Through Scripting Matt Williams Evangelist @ Datadog Organizer

    @ DevOpsDays Boston 2017 @technovangelist http://dtdg.co/nginx17deck
  2. @technovangelist nginx_status NGINX NGINX Plus stub_status_module http_status_module location /nginx_status {

    stub_status; } server { location / { health_check; } status_zone server_backend; location /nginx_status { status; } }
  3. @technovangelist {"version":8,"nginx_version":"1.11.10","nginx_build":"nginx-plus-r12-p3","address":"206.251.255.64","generation":5,"load_timestamp": 1501322401673,"timestamp":1501861405424,"pid":26474,"ppid":96917,"processes":{"respawned":0},"connections":{"accepted":23164115,"dropp 752731,"active":9,"idle":386},"ssl":{"handshakes":255730,"handshakes_failed":41151,"session_reuses":35906},"requests":{"total": 48895013,"current":7},"server_zones":{"hg.nginx.org":{"processing":1,"requests":1310186,"responses":{"1xx":0,"2xx":1293954,"3xx":12629 2419,"5xx":993,"total":1309995},"discarded":190,"received":325707828,"sent":38155544995},"trac.nginx.org":{"processing":5,"requests": 445206,"responses":{"1xx":0,"2xx":273454,"3xx":91942,"4xx":7008,"5xx":21013,"total":393417},"discarded":51784,"received":123051863,"se 11514166530},"lxr.nginx.org":{"processing":0,"requests":8488,"responses":{"1xx":0,"2xx":6743,"3xx":406,"4xx":556,"5xx":490,"total": 8195},"discarded":293,"received":3342520,"sent":280101100}},"slabs":{"bot":{"pages":{"used":2,"free":29},"slots":{"8":{"used":0,"free" 0,"reqs":0,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":1,"free":126,"reqs":1,"fails":0},"64":{"used":0,"free":

    0,"reqs":0,"fails":0},"128":{"used":10,"free":22,"reqs":375,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"512":{"used":0,"f 0,"reqs":0,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}},"ban":{"pages":{"u 2,"free":29},"slots":{"8":{"used":0,"free":0,"reqs":0,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":1,"free": 126,"reqs":1,"fails":0},"64":{"used":0,"free":0,"reqs":0,"fails":0},"128":{"used":1,"free":31,"reqs":1,"fails":0},"256":{"used":0,"fre 0,"reqs":0,"fails":0},"512":{"used":0,"free":0,"reqs":0,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"fre 0,"reqs":0,"fails":0}}},"one":{"pages":{"used":3,"free":251},"slots":{"8":{"used":0,"free":0,"reqs":0,"fails":0},"16":{"used":0,"free" 0,"reqs":0,"fails":0},"32":{"used":1,"free":126,"reqs":1,"fails":0},"64":{"used":0,"free":0,"reqs":0,"fails":0},"128":{"used":17,"free 47,"reqs":407166,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"512":{"used":0,"free":0,"reqs":0,"fails":0},"1024":{"used": 0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}},"http_cache":{"pages":{"used":1024,"free":3047},"slots" {"used":0,"free":0,"reqs":0,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":0,"free":0,"reqs":0,"fails":0},"64":{" 1,"free":63,"reqs":1,"fails":0},"128":{"used":0,"free":0,"reqs":0,"fails":0},"256":{"used":16284,"free":68,"reqs":6557387,"fails":0}," {"used":1,"free":7,"reqs":1,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}}," {"pages":{"used":1175,"free":11550},"slots":{"8":{"used":0,"free":0,"reqs":0,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"3 {"used":0,"free":0,"reqs":0,"fails":0},"64":{"used":1,"free":63,"reqs":1,"fails":0},"128":{"used":7260,"free":5412,"reqs":181010,"fail 0},"256":{"used":7259,"free":5189,"reqs":180672,"fails":0},"512":{"used":0,"free":0,"reqs":0,"fails":0},"1024":{"used":0,"free":0,"req 4,"fails":0},"2048":{"used":0,"free":0,"reqs":332,"fails":0}}},"trac-backend":{"pages":{"used":5,"free":2},"slots":{"8":{"used":1,"fre 503,"reqs":1,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":4,"free":123,"reqs":4,"fails":0},"64":{"used":1,"free 63,"reqs":1,"fails":0},"128":{"used":6,"free":26,"reqs":6,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"512":{"used":2,"fre 6,"reqs":2,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}},"hg-backend":{"pag {"used":4,"free":3},"slots":{"8":{"used":1,"free":503,"reqs":1,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":5," 122,"reqs":5,"fails":0},"64":{"used":0,"free":0,"reqs":0,"fails":0},"128":{"used":6,"free":26,"reqs":6,"fails":0},"256":{"used":0,"fre 0,"reqs":0,"fails":0},"512":{"used":2,"free":6,"reqs":2,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"fre 0,"reqs":0,"fails":0}}},"lxr-backend":{"pages":{"used":4,"free":3},"slots":{"8":{"used":1,"free":503,"reqs":1,"fails":0},"16":{"used": 0,"free":0,"reqs":0,"fails":0},"32":{"used":5,"free":122,"reqs":5,"fails":0},"64":{"used":0,"free":0,"reqs":0,"fails":0},"128":{"used" 6,"free":26,"reqs":6,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"512":{"used":2,"free":6,"reqs":2,"fails":0},"1024":{"use 0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}},"demo-backend":{"pages":{"used":5,"free":2},"slots":{"8 {"used":1,"free":503,"reqs":1,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":2,"free":125,"reqs":2,"fails":0},"64 {"used":1,"free":63,"reqs":1,"fails":0},"128":{"used":4,"free":28,"reqs":4,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"51
  4. @technovangelist NGINX Plus metrics nginx.net.writing nginx.net.waiting nginx.net.reading nginx.connections.accepted nginx.connections.active nginx.connections.dropped

    nginx.connections.idle nginx.generation nginx.load_timestamp nginx.pid nginx.processes.respawned nginx.requests.current nginx.requests.total nginx.server_zone.discarded nginx.server_zone.processing nginx.server_zone.received nginx.server_zone.requests nginx.server_zone.responses.1xx nginx.server_zone.responses.2xx nginx.server_zone.responses.3xx nginx.server_zone.responses.4xx nginx.server_zone.responses.5xx nginx.server_zone.responses.total nginx.server_zone.sent nginx.ssl.handshakes nginx.ssl.handshakes_failed nginx.ssl.session_reuses nginx.timestamp nginx.upstream.keepalive nginx.upstream.peers.active nginx.upstream.peers.backup nginx.upstream.peers.downstart nginx.upstream.peers.downtime nginx.upstream.peers.fails nginx.upstream.peers.id nginx.upstream.peers.received nginx.upstream.peers.requests nginx.upstream.peers.responses. nginx.upstream.peers.selected nginx.upstream.peers.sent nginx.upstream.peers.weight nginx.version
  5. @technovangelist How to get more metrics •Watch an aggregated stats

    view • nginx_status • Create your own •Let your log aggregation tool create stats • Configure your log output •Send each request’s stats
  6. @technovangelist $ancient_browser $arg_ $args $binary_remote_addr $body_bytes_sent $bytes_received $bytes_sent $connection $connection_requests

    $connections_active $connections_reading $connections_waiting $connections_writing $content_length $content_type $cookie_ $date_gmt $date_local $geoip_region $geoip_region_name $gzip_ratio $host $hostname $http2 $http_ $https $invalid_referer $is_args $jwt_claim_ $jwt_header_ $limit_rate $memcached_key $modern_browser $msec $msie $nginx_version $request_body $request_body_file $request_completion $request_filename $request_id $request_length $request_method $request_time $request_uri $scheme $secure_link $secure_link_expires $sent_http_ $sent_trailer_ $server_addr $server_name $server_port $server_protocol $ssl_client_v_end $ssl_client_v_remain $ssl_client_v_start $ssl_client_verify $ssl_curves $ssl_preread_server_nam $ssl_protocol $ssl_server_name $ssl_session_id $ssl_session_reused $status $tcpinfo_rtt $tcpinfo_rttvar $tcpinfo_snd_cwnd $tcpinfo_rcv_space $time_iso8601 $time_local $uid_got Available NGINX Variables
  7. @technovangelist Sample Log Config http { log_format compression '$remote_addr -

    $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent" "$gzip_ratio"'; server { gzip on; access_log /spool/logs/nginx-access.log compression; ... } }
  8. @technovangelist Scripting Choices •lua - OpenResty • Easy to learn

    lua (http://dtdg.co/nginx17-lua) • More mature •nginScript • You may already know Javascript • Created by NGINX
  9. @technovangelist Scripting Benefits •Per-request config file • Different log file

    config per: • Set of pages • Set of users •Provide additional functionality • Sub-requests • Reloader - http://dtdg.co/nginx17-reload
  10. @technovangelist Scripting Gotchas •Scripting environment flushed per request •Lua can

    get around this • Buckets • Sub requests •nginScript maybe later this year
  11. @technovangelist Aggregated Stats - luameter.com { "connections": {"active": "25","reading": "0","waiting":

    "24","writing": "1"}, "luameter_version": "1.0", "meta_version": "97fe7f", "nginx_version": "1.7.9", "ngx_lua_version": "0.9.13", "timestamp": 1502896390, "upstreams": { "127.0.0.1:8080": { "cache": {"bypass": 0,"expired": 0,"hit": 0,"miss": 0,"revalidated": 0,"stal "latency": {"max": 0,"mean": 0,"median": 0,"min": 0,"p75": 0,"p90": 0,"p95": "rates": {"fifteen_minute_avg": 4.45e-322,"five_minute_avg": 1.5e-322,"mean" "received": 23281719, "requests": 23281719, "responses": {"200": 23281718,"1xx": 0,"2xx": 23281718,"3xx": 0,"4xx": 0,"5x "sent": 46563436 }, "127.0.0.1:8081": { "cache": {"bypass": 0,"expired": 0,"hit": 0,"miss": 0,"revalidated": 0,"stal "latency": {"max": 0,"mean": 0,"median": 0,"min": 0,"p75": 0,"p90": 0,"p95":
  12. @technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers

    kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } } https://www.nginx.com/blog/introduction-nginscript/
  13. @technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers

    kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } } https://www.nginx.com/blog/introduction-nginscript/
  14. @technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers

    kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } } https://www.nginx.com/blog/introduction-nginscript/
  15. @technovangelist function kvAccessLog(req, res) { var log = req.variables.time_iso8601; log

    += " client=" + req.remoteAddress; log += " method=" + req.method; log += " uri=" + req.uri; log += " status=" + res.status; log += kvHeaders(req.headers, "req"); log += kvHeaders(res.headers, "res"); return log; } https://www.nginx.com/blog/introduction-nginscript/
  16. @technovangelist function kvAccessLog(req, res) { var log = req.variables.time_iso8601; log

    += " client=" + req.remoteAddress; log += " method=" + req.method; log += " uri=" + req.uri; log += " status=" + res.status; log += kvHeaders(req.headers, "req"); log += kvHeaders(res.headers, "res"); return log; } https://www.nginx.com/blog/introduction-nginscript/
  17. @technovangelist function kvHeaders(headers, parent) { var kvpairs = ""; for

    (var h in headers) { kvpairs += " " + parent + "." + h + "="; if ( headers[h].indexOf(" ") == -1 ) { kvpairs += headers[h]; } else { kvpairs += "'" + headers[h] + "'"; } } return kvpairs; } https://www.nginx.com/blog/introduction-nginscript/
  18. @technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers

    kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } }
  19. @technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers

    kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } }
  20. @technovangelist ngx_lua_datadog location { local conf = { host =

    "127.0.0.1", port = 8125, namespace = "Stats", timeout = 1} local statsd_logger = require "ngx_lua_datadog" local logger, err = statsd_logger:new(conf) if err then ngx_log(ngx.ERR, "failed to create Statsd logger: ", err) end if ( my_condition ) then logger:counter("condition_matched", 1, 1) ngx.exit(204) else logger:counter("condition_not_matched", 1, 1) ngx.exit(200) end } http://dtdg.co/nginx17-luadatadog
  21. @technovangelist ngx_lua_datadog location { local conf = { host =

    "127.0.0.1", port = 8125, namespace = "Stats", timeout = 1} local statsd_logger = require "ngx_lua_datadog" local logger, err = statsd_logger:new(conf) if err then ngx_log(ngx.ERR, "failed to create Statsd logger: ", err) end if ( my_condition ) then logger:counter("condition_matched", 1, 1) ngx.exit(204) else logger:counter("condition_not_matched", 1, 1) ngx.exit(200) end } http://dtdg.co/nginx17-luadatadog
  22. @technovangelist ngx_lua_datadog location { local conf = { host =

    "127.0.0.1", port = 8125, namespace = "Stats", timeout = 1} local statsd_logger = require "ngx_lua_datadog" local logger, err = statsd_logger:new(conf) if err then ngx_log(ngx.ERR, "failed to create Statsd logger: ", err) end if ( my_condition ) then logger:counter("condition_matched", 1, 1) ngx.exit(204) else logger:counter("condition_not_matched", 1, 1) ngx.exit(200) end } http://dtdg.co/nginx17-luadatadog
  23. @technovangelist ngx_lua_datadog function new(conf) local sock = ngx.socket.udp() sock:settimeout(conf.timeout) local

    _, err = sock:setpeername(conf.host, conf.port) if err then return nil, "failed to connect to "..conf.host..":"..conf.port..": "..err end local dogstatsd = { host = conf.host, port = conf.port, socket = sock, namespace = conf.namespace } return setmetatable(dogstatsd, dogstatsd_mt) end http://dtdg.co/nginx17-luadatadog
  24. @technovangelist ngx_lua_datadog location { local conf = { host =

    "127.0.0.1", port = 8125, namespace = "Stats", timeout = 1} local statsd_logger = require "ngx_lua_datadog" local logger, err = statsd_logger:new(conf) if err then ngx_log(ngx.ERR, "failed to create Statsd logger: ", err) end if ( my_condition ) then logger:counter("condition_matched", 1, 1) ngx.exit(204) else logger:counter("condition_not_matched", 1, 1) ngx.exit(200) end } http://dtdg.co/nginx17-luadatadog
  25. @technovangelist ngx_lua_datadog function counter(stat, value, sample_rate, tags) return self:send_dogstatsd(stat, value,

    "c", sample_rate, tags) end function send_dogstatsd(stat, delta, kind, sample_rate, tags) local udp_message = self:create_dogstatsd_message(stat, delta, kind, sample_rate, tags) local ok, err = self.socket:send(udp_message) if not ok then ngx.log(ngx.ERR, "failed to send data to ………………, err) end end http://dtdg.co/nginx17-luadatadog
  26. @technovangelist Use nginx-dogstatsd server { listen 80; server_name www.your.domain.com; dogstatsd_count

    "your_product.requests" 1; location / { dogstatsd_count "your_product.pages.index_requests" 1; dogstatsd_count “your_prod…ex_requests” 1 "server:$server_name"; dogstatsd_count “your_prod…ex_responses” 1 "" "$request_completion"; dogstatsd_timing “your_prod…ex_response_time” "$upstream_response_time"; dogstatsd_count “your_prod…om_$upstream_http_x_some_custom_header” 1 "" "$upstream_http_x_some_custom_header"; proxy_pass http://some.other.domain.com; } } http://dtdg.co/nginx17-dogstatsd
  27. @technovangelist Use nginx-dogstatsd / nginScript js_include conf.d/metrics.js; js_set $us_requests usrequests;

    js_set $uk_requests ukrequests; js_set $dnt_requests dntrequests; js_set $process_time proctime; server { listen 80; dogstatsd_count "product.requests.usa" $us_requests; dogstatsd_count "product.requests.uk" $uk_requests; dogstatds__count "do_not_track.requests" $dnt_requests; dogstatsd_timing "product.requests.timing" $process_time; ... } http://dtdg.co/nginx17-dogstatsd
  28. @technovangelist Review •Scripting makes it possible to get better metrics

    •Can use Lua or nginScript •Lua is more functional today, nginScript has potential
  29. @technovangelist Other cool things to do Dev Essentials lua-resty-cookie Routing

    Libraries Middleware Kong Templating Liquid Validation Authentication Cryptography Auto SSL with Let’s Encrypt Network Database / Storage mysql, postgres, mongo Profiling Flame Graph MQ Kafka Bar/QR Code Compression Image Formats Caching Logging Awesome Resty - http://dtdg.co/nginx17-awesome
  30. @technovangelist Finding more information The Latest and Greatest from ngx_lua:

    New Features & Tools http://dtdg.co/ddnginx-vid1 A New and Powerful Way to Configure NGINX, nginScript
 http://dtdg.co/ddnginx-vid3